1 /*
   2  * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/cardTable.hpp"
  30 #include "gc/shared/cardTableBarrierSet.inline.hpp"
  31 #include "gc/shared/collectedHeap.hpp"
  32 #include "interp_masm_arm.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "interpreter/interpreterRuntime.hpp"
  35 #include "logging/log.hpp"
  36 #include "oops/arrayOop.hpp"
  37 #include "oops/markWord.hpp"
  38 #include "oops/method.hpp"
  39 #include "oops/methodData.hpp"
  40 #include "prims/jvmtiExport.hpp"
  41 #include "prims/jvmtiThreadState.hpp"
  42 #include "runtime/basicLock.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/safepointMechanism.hpp"
  45 #include "runtime/sharedRuntime.hpp"
  46 #include "utilities/powerOfTwo.hpp"
  47 
  48 //--------------------------------------------------------------------
  49 // Implementation of InterpreterMacroAssembler
  50 
  51 
  52 
  53 
  54 InterpreterMacroAssembler::InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) {
  55 }
  56 
  57 void InterpreterMacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
  58 #ifdef ASSERT
  59   // Ensure that last_sp is not filled.
  60   { Label L;
  61     ldr(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
  62     cbz(Rtemp, L);
  63     stop("InterpreterMacroAssembler::call_VM_helper: last_sp != NULL");
  64     bind(L);
  65   }
  66 #endif // ASSERT
  67 
  68   // Rbcp must be saved/restored since it may change due to GC.
  69   save_bcp();
  70 
  71 
  72   // super call
  73   MacroAssembler::call_VM_helper(oop_result, entry_point, number_of_arguments, check_exceptions);
  74 
  75 
  76   // Restore interpreter specific registers.
  77   restore_bcp();
  78   restore_method();
  79 }
  80 
  81 void InterpreterMacroAssembler::jump_to_entry(address entry) {
  82   assert(entry, "Entry must have been generated by now");
  83   b(entry);
  84 }
  85 
  86 void InterpreterMacroAssembler::check_and_handle_popframe() {
  87   if (can_pop_frame()) {
  88     Label L;
  89     const Register popframe_cond = R2_tmp;
  90 
  91     // Initiate popframe handling only if it is not already being processed.  If the flag
  92     // has the popframe_processing bit set, it means that this code is called *during* popframe
  93     // handling - we don't want to reenter.
  94 
  95     ldr_s32(popframe_cond, Address(Rthread, JavaThread::popframe_condition_offset()));
  96     tbz(popframe_cond, exact_log2(JavaThread::popframe_pending_bit), L);
  97     tbnz(popframe_cond, exact_log2(JavaThread::popframe_processing_bit), L);
  98 
  99     // Call Interpreter::remove_activation_preserving_args_entry() to get the
 100     // address of the same-named entrypoint in the generated interpreter code.
 101     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));
 102 
 103     // Call indirectly to avoid generation ordering problem.
 104     jump(R0);
 105 
 106     bind(L);
 107   }
 108 }
 109 
 110 
 111 // Blows R2, Rtemp. Sets TOS cached value.
 112 void InterpreterMacroAssembler::load_earlyret_value(TosState state) {
 113   const Register thread_state = R2_tmp;
 114 
 115   ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
 116 
 117   const Address tos_addr(thread_state, JvmtiThreadState::earlyret_tos_offset());
 118   const Address oop_addr(thread_state, JvmtiThreadState::earlyret_oop_offset());
 119   const Address val_addr(thread_state, JvmtiThreadState::earlyret_value_offset());
 120   const Address val_addr_hi(thread_state, JvmtiThreadState::earlyret_value_offset()
 121                              + in_ByteSize(wordSize));
 122 
 123   Register zero = zero_register(Rtemp);
 124 
 125   switch (state) {
 126     case atos: ldr(R0_tos, oop_addr);
 127                str(zero, oop_addr);
 128                interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 129                break;
 130 
 131     case ltos: ldr(R1_tos_hi, val_addr_hi);        // fall through
 132     case btos:                                     // fall through
 133     case ztos:                                     // fall through
 134     case ctos:                                     // fall through
 135     case stos:                                     // fall through
 136     case itos: ldr_s32(R0_tos, val_addr);          break;
 137 #ifdef __SOFTFP__
 138     case dtos: ldr(R1_tos_hi, val_addr_hi);        // fall through
 139     case ftos: ldr(R0_tos, val_addr);              break;
 140 #else
 141     case ftos: ldr_float (S0_tos, val_addr);       break;
 142     case dtos: ldr_double(D0_tos, val_addr);       break;
 143 #endif // __SOFTFP__
 144     case vtos: /* nothing to do */                 break;
 145     default  : ShouldNotReachHere();
 146   }
 147   // Clean up tos value in the thread object
 148   str(zero, val_addr);
 149   str(zero, val_addr_hi);
 150 
 151   mov(Rtemp, (int) ilgl);
 152   str_32(Rtemp, tos_addr);
 153 }
 154 
 155 
 156 // Blows R2, Rtemp.
 157 void InterpreterMacroAssembler::check_and_handle_earlyret() {
 158   if (can_force_early_return()) {
 159     Label L;
 160     const Register thread_state = R2_tmp;
 161 
 162     ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
 163     cbz(thread_state, L); // if (thread->jvmti_thread_state() == NULL) exit;
 164 
 165     // Initiate earlyret handling only if it is not already being processed.
 166     // If the flag has the earlyret_processing bit set, it means that this code
 167     // is called *during* earlyret handling - we don't want to reenter.
 168 
 169     ldr_s32(Rtemp, Address(thread_state, JvmtiThreadState::earlyret_state_offset()));
 170     cmp(Rtemp, JvmtiThreadState::earlyret_pending);
 171     b(L, ne);
 172 
 173     // Call Interpreter::remove_activation_early_entry() to get the address of the
 174     // same-named entrypoint in the generated interpreter code.
 175 
 176     ldr_s32(R0, Address(thread_state, JvmtiThreadState::earlyret_tos_offset()));
 177     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), R0);
 178 
 179     jump(R0);
 180 
 181     bind(L);
 182   }
 183 }
 184 
 185 
 186 // Sets reg. Blows Rtemp.
 187 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) {
 188   assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");
 189   assert(reg != Rtemp, "should be different registers");
 190 
 191   ldrb(Rtemp, Address(Rbcp, bcp_offset));
 192   ldrb(reg, Address(Rbcp, bcp_offset+1));
 193   orr(reg, reg, AsmOperand(Rtemp, lsl, BitsPerByte));
 194 }
 195 
 196 void InterpreterMacroAssembler::get_index_at_bcp(Register index, int bcp_offset, Register tmp_reg, size_t index_size) {
 197   assert_different_registers(index, tmp_reg);
 198   if (index_size == sizeof(u2)) {
 199     // load bytes of index separately to avoid unaligned access
 200     ldrb(index, Address(Rbcp, bcp_offset+1));
 201     ldrb(tmp_reg, Address(Rbcp, bcp_offset));
 202     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 203   } else if (index_size == sizeof(u4)) {
 204     ldrb(index, Address(Rbcp, bcp_offset+3));
 205     ldrb(tmp_reg, Address(Rbcp, bcp_offset+2));
 206     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 207     ldrb(tmp_reg, Address(Rbcp, bcp_offset+1));
 208     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 209     ldrb(tmp_reg, Address(Rbcp, bcp_offset));
 210     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 211     // Check if the secondary index definition is still ~x, otherwise
 212     // we have to change the following assembler code to calculate the
 213     // plain index.
 214     assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line");
 215     mvn_32(index, index);  // convert to plain index
 216   } else if (index_size == sizeof(u1)) {
 217     ldrb(index, Address(Rbcp, bcp_offset));
 218   } else {
 219     ShouldNotReachHere();
 220   }
 221 }
 222 
 223 // Sets cache, index.
 224 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size) {
 225   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 226   assert_different_registers(cache, index);
 227 
 228   get_index_at_bcp(index, bcp_offset, cache, index_size);
 229 
 230   // load constant pool cache pointer
 231   ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));
 232 
 233   // convert from field index to ConstantPoolCacheEntry index
 234   assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
 235   logical_shift_left(index, index, 2);
 236 }
 237 
 238 // Sets cache, index, bytecode.
 239 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size) {
 240   get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size);
 241   // caution index and bytecode can be the same
 242   add(bytecode, cache, AsmOperand(index, lsl, LogBytesPerWord));
 243   ldrb(bytecode, Address(bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset())));
 244   TemplateTable::volatile_barrier(MacroAssembler::LoadLoad, noreg, true);
 245 }
 246 
 247 // Sets cache. Blows reg_tmp.
 248 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register reg_tmp, int bcp_offset, size_t index_size) {
 249   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 250   assert_different_registers(cache, reg_tmp);
 251 
 252   get_index_at_bcp(reg_tmp, bcp_offset, cache, index_size);
 253 
 254   // load constant pool cache pointer
 255   ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));
 256 
 257   // skip past the header
 258   add(cache, cache, in_bytes(ConstantPoolCache::base_offset()));
 259   // convert from field index to ConstantPoolCacheEntry index
 260   // and from word offset to byte offset
 261   assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
 262   add(cache, cache, AsmOperand(reg_tmp, lsl, 2 + LogBytesPerWord));
 263 }
 264 
 265 // Load object from cpool->resolved_references(index)
 266 void InterpreterMacroAssembler::load_resolved_reference_at_index(
 267                                            Register result, Register index) {
 268   assert_different_registers(result, index);
 269   get_constant_pool(result);
 270 
 271   Register cache = result;
 272   // load pointer for resolved_references[] objArray
 273   ldr(cache, Address(result, ConstantPool::cache_offset_in_bytes()));
 274   ldr(cache, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes()));
 275   resolve_oop_handle(cache);
 276   // Add in the index
 277   // convert from field index to resolved_references() index and from
 278   // word index to byte offset. Since this is a java object, it can be compressed
 279   logical_shift_left(index, index, LogBytesPerHeapOop);
 280   add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
 281   load_heap_oop(result, Address(cache, index));
 282 }
 283 
 284 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
 285                                            Register Rcpool, Register Rindex, Register Rklass) {
 286   add(Rtemp, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord));
 287   ldrh(Rtemp, Address(Rtemp, sizeof(ConstantPool))); // Rtemp = resolved_klass_index
 288   ldr(Rklass, Address(Rcpool,  ConstantPool::resolved_klasses_offset_in_bytes())); // Rklass = cpool->_resolved_klasses
 289   add(Rklass, Rklass, AsmOperand(Rtemp, lsl, LogBytesPerWord));
 290   ldr(Rklass, Address(Rklass, Array<Klass*>::base_offset_in_bytes()));
 291 }
 292 
 293 // Generate a subtype check: branch to not_subtype if sub_klass is
 294 // not a subtype of super_klass.
 295 // Profiling code for the subtype check failure (profile_typecheck_failed)
 296 // should be explicitly generated by the caller in the not_subtype case.
 297 // Blows Rtemp, tmp1, tmp2.
 298 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
 299                                                   Register Rsuper_klass,
 300                                                   Label &not_subtype,
 301                                                   Register tmp1,
 302                                                   Register tmp2) {
 303 
 304   assert_different_registers(Rsub_klass, Rsuper_klass, tmp1, tmp2, Rtemp);
 305   Label ok_is_subtype, loop, update_cache;
 306 
 307   const Register super_check_offset = tmp1;
 308   const Register cached_super = tmp2;
 309 
 310   // Profile the not-null value's klass.
 311   profile_typecheck(tmp1, Rsub_klass);
 312 
 313   // Load the super-klass's check offset into
 314   ldr_u32(super_check_offset, Address(Rsuper_klass, Klass::super_check_offset_offset()));
 315 
 316   // Check for self
 317   cmp(Rsub_klass, Rsuper_klass);
 318 
 319   // Load from the sub-klass's super-class display list, or a 1-word cache of
 320   // the secondary superclass list, or a failing value with a sentinel offset
 321   // if the super-klass is an interface or exceptionally deep in the Java
 322   // hierarchy and we have to scan the secondary superclass list the hard way.
 323   // See if we get an immediate positive hit
 324   ldr(cached_super, Address(Rsub_klass, super_check_offset));
 325 
 326   cond_cmp(Rsuper_klass, cached_super, ne);
 327   b(ok_is_subtype, eq);
 328 
 329   // Check for immediate negative hit
 330   cmp(super_check_offset, in_bytes(Klass::secondary_super_cache_offset()));
 331   b(not_subtype, ne);
 332 
 333   // Now do a linear scan of the secondary super-klass chain.
 334   const Register supers_arr = tmp1;
 335   const Register supers_cnt = tmp2;
 336   const Register cur_super  = Rtemp;
 337 
 338   // Load objArrayOop of secondary supers.
 339   ldr(supers_arr, Address(Rsub_klass, Klass::secondary_supers_offset()));
 340 
 341   ldr_u32(supers_cnt, Address(supers_arr, Array<Klass*>::length_offset_in_bytes())); // Load the array length
 342   cmp(supers_cnt, 0);
 343 
 344   // Skip to the start of array elements and prefetch the first super-klass.
 345   ldr(cur_super, Address(supers_arr, Array<Klass*>::base_offset_in_bytes(), pre_indexed), ne);
 346   b(not_subtype, eq);
 347 
 348   bind(loop);
 349 
 350 
 351   cmp(cur_super, Rsuper_klass);
 352   b(update_cache, eq);
 353 
 354   subs(supers_cnt, supers_cnt, 1);
 355 
 356   ldr(cur_super, Address(supers_arr, wordSize, pre_indexed), ne);
 357 
 358   b(loop, ne);
 359 
 360   b(not_subtype);
 361 
 362   bind(update_cache);
 363   // Must be equal but missed in cache.  Update cache.
 364   str(Rsuper_klass, Address(Rsub_klass, Klass::secondary_super_cache_offset()));
 365 
 366   bind(ok_is_subtype);
 367 }
 368 
 369 
 370 //////////////////////////////////////////////////////////////////////////////////
 371 
 372 
 373 // Java Expression Stack
 374 
 375 void InterpreterMacroAssembler::pop_ptr(Register r) {
 376   assert(r != Rstack_top, "unpredictable instruction");
 377   ldr(r, Address(Rstack_top, wordSize, post_indexed));
 378 }
 379 
 380 void InterpreterMacroAssembler::pop_i(Register r) {
 381   assert(r != Rstack_top, "unpredictable instruction");
 382   ldr_s32(r, Address(Rstack_top, wordSize, post_indexed));
 383   zap_high_non_significant_bits(r);
 384 }
 385 
 386 void InterpreterMacroAssembler::pop_l(Register lo, Register hi) {
 387   assert_different_registers(lo, hi);
 388   assert(lo < hi, "lo must be < hi");
 389   pop(RegisterSet(lo) | RegisterSet(hi));
 390 }
 391 
 392 void InterpreterMacroAssembler::pop_f(FloatRegister fd) {
 393   fpops(fd);
 394 }
 395 
 396 void InterpreterMacroAssembler::pop_d(FloatRegister fd) {
 397   fpopd(fd);
 398 }
 399 
 400 
 401 // Transition vtos -> state. Blows R0, R1. Sets TOS cached value.
 402 void InterpreterMacroAssembler::pop(TosState state) {
 403   switch (state) {
 404     case atos: pop_ptr(R0_tos);                              break;
 405     case btos:                                               // fall through
 406     case ztos:                                               // fall through
 407     case ctos:                                               // fall through
 408     case stos:                                               // fall through
 409     case itos: pop_i(R0_tos);                                break;
 410     case ltos: pop_l(R0_tos_lo, R1_tos_hi);                  break;
 411 #ifdef __SOFTFP__
 412     case ftos: pop_i(R0_tos);                                break;
 413     case dtos: pop_l(R0_tos_lo, R1_tos_hi);                  break;
 414 #else
 415     case ftos: pop_f(S0_tos);                                break;
 416     case dtos: pop_d(D0_tos);                                break;
 417 #endif // __SOFTFP__
 418     case vtos: /* nothing to do */                           break;
 419     default  : ShouldNotReachHere();
 420   }
 421   interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 422 }
 423 
 424 void InterpreterMacroAssembler::push_ptr(Register r) {
 425   assert(r != Rstack_top, "unpredictable instruction");
 426   str(r, Address(Rstack_top, -wordSize, pre_indexed));
 427   check_stack_top_on_expansion();
 428 }
 429 
 430 void InterpreterMacroAssembler::push_i(Register r) {
 431   assert(r != Rstack_top, "unpredictable instruction");
 432   str_32(r, Address(Rstack_top, -wordSize, pre_indexed));
 433   check_stack_top_on_expansion();
 434 }
 435 
 436 void InterpreterMacroAssembler::push_l(Register lo, Register hi) {
 437   assert_different_registers(lo, hi);
 438   assert(lo < hi, "lo must be < hi");
 439   push(RegisterSet(lo) | RegisterSet(hi));
 440 }
 441 
 442 void InterpreterMacroAssembler::push_f() {
 443   fpushs(S0_tos);
 444 }
 445 
 446 void InterpreterMacroAssembler::push_d() {
 447   fpushd(D0_tos);
 448 }
 449 
 450 // Transition state -> vtos. Blows Rtemp.
 451 void InterpreterMacroAssembler::push(TosState state) {
 452   interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 453   switch (state) {
 454     case atos: push_ptr(R0_tos);                              break;
 455     case btos:                                                // fall through
 456     case ztos:                                                // fall through
 457     case ctos:                                                // fall through
 458     case stos:                                                // fall through
 459     case itos: push_i(R0_tos);                                break;
 460     case ltos: push_l(R0_tos_lo, R1_tos_hi);                  break;
 461 #ifdef __SOFTFP__
 462     case ftos: push_i(R0_tos);                                break;
 463     case dtos: push_l(R0_tos_lo, R1_tos_hi);                  break;
 464 #else
 465     case ftos: push_f();                                      break;
 466     case dtos: push_d();                                      break;
 467 #endif // __SOFTFP__
 468     case vtos: /* nothing to do */                            break;
 469     default  : ShouldNotReachHere();
 470   }
 471 }
 472 
 473 
 474 
 475 // Converts return value in R0/R1 (interpreter calling conventions) to TOS cached value.
 476 void InterpreterMacroAssembler::convert_retval_to_tos(TosState state) {
 477 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
 478   // According to interpreter calling conventions, result is returned in R0/R1,
 479   // but templates expect ftos in S0, and dtos in D0.
 480   if (state == ftos) {
 481     fmsr(S0_tos, R0);
 482   } else if (state == dtos) {
 483     fmdrr(D0_tos, R0, R1);
 484   }
 485 #endif // !__SOFTFP__ && !__ABI_HARD__
 486 }
 487 
 488 // Converts TOS cached value to return value in R0/R1 (according to interpreter calling conventions).
 489 void InterpreterMacroAssembler::convert_tos_to_retval(TosState state) {
 490 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
 491   // According to interpreter calling conventions, result is returned in R0/R1,
 492   // so ftos (S0) and dtos (D0) are moved to R0/R1.
 493   if (state == ftos) {
 494     fmrs(R0, S0_tos);
 495   } else if (state == dtos) {
 496     fmrrd(R0, R1, D0_tos);
 497   }
 498 #endif // !__SOFTFP__ && !__ABI_HARD__
 499 }
 500 
 501 
 502 
 503 // Helpers for swap and dup
 504 void InterpreterMacroAssembler::load_ptr(int n, Register val) {
 505   ldr(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n)));
 506 }
 507 
 508 void InterpreterMacroAssembler::store_ptr(int n, Register val) {
 509   str(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n)));
 510 }
 511 
 512 
 513 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() {
 514 
 515   // set sender sp
 516   mov(Rsender_sp, SP);
 517 
 518   // record last_sp
 519   str(Rsender_sp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
 520 }
 521 
 522 // Jump to from_interpreted entry of a call unless single stepping is possible
 523 // in this thread in which case we must call the i2i entry
 524 void InterpreterMacroAssembler::jump_from_interpreted(Register method) {
 525   assert_different_registers(method, Rtemp);
 526 
 527   prepare_to_jump_from_interpreted();
 528 
 529   if (can_post_interpreter_events()) {
 530     // JVMTI events, such as single-stepping, are implemented partly by avoiding running
 531     // compiled code in threads for which the event is enabled.  Check here for
 532     // interp_only_mode if these events CAN be enabled.
 533 
 534     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
 535     cmp(Rtemp, 0);
 536     ldr(PC, Address(method, Method::interpreter_entry_offset()), ne);
 537   }
 538 
 539   indirect_jump(Address(method, Method::from_interpreted_offset()), Rtemp);
 540 }
 541 
 542 
 543 void InterpreterMacroAssembler::restore_dispatch() {
 544   mov_slow(RdispatchTable, (address)Interpreter::dispatch_table(vtos));
 545 }
 546 
 547 
 548 // The following two routines provide a hook so that an implementation
 549 // can schedule the dispatch in two parts.
 550 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) {
 551   // Nothing ARM-specific to be done here.
 552 }
 553 
 554 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {
 555   dispatch_next(state, step);
 556 }
 557 
 558 void InterpreterMacroAssembler::dispatch_base(TosState state,
 559                                               DispatchTableMode table_mode,
 560                                               bool verifyoop, bool generate_poll) {
 561   if (VerifyActivationFrameSize) {
 562     Label L;
 563     sub(Rtemp, FP, SP);
 564     int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize;
 565     cmp(Rtemp, min_frame_size);
 566     b(L, ge);
 567     stop("broken stack frame");
 568     bind(L);
 569   }
 570 
 571   if (verifyoop) {
 572     interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 573   }
 574 
 575   Label safepoint;
 576   address* const safepoint_table = Interpreter::safept_table(state);
 577   address* const table           = Interpreter::dispatch_table(state);
 578   bool needs_thread_local_poll = generate_poll && table != safepoint_table;
 579 
 580   if (needs_thread_local_poll) {
 581     NOT_PRODUCT(block_comment("Thread-local Safepoint poll"));
 582     ldr(Rtemp, Address(Rthread, JavaThread::polling_word_offset()));
 583     tbnz(Rtemp, exact_log2(SafepointMechanism::poll_bit()), safepoint);
 584   }
 585 
 586   if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) {
 587     zap_high_non_significant_bits(R0_tos);
 588   }
 589 
 590 #ifdef ASSERT
 591   Label L;
 592   mov_slow(Rtemp, (address)Interpreter::dispatch_table(vtos));
 593   cmp(Rtemp, RdispatchTable);
 594   b(L, eq);
 595   stop("invalid RdispatchTable");
 596   bind(L);
 597 #endif
 598 
 599   if (table_mode == DispatchDefault) {
 600     if (state == vtos) {
 601       indirect_jump(Address::indexed_ptr(RdispatchTable, R3_bytecode), Rtemp);
 602     } else {
 603       // on 32-bit ARM this method is faster than the one above.
 604       sub(Rtemp, RdispatchTable, (Interpreter::distance_from_dispatch_table(vtos) -
 605                            Interpreter::distance_from_dispatch_table(state)) * wordSize);
 606       indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
 607     }
 608   } else {
 609     assert(table_mode == DispatchNormal, "invalid dispatch table mode");
 610     address table = (address) Interpreter::normal_table(state);
 611     mov_slow(Rtemp, table);
 612     indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
 613   }
 614 
 615   if (needs_thread_local_poll) {
 616     bind(safepoint);
 617     lea(Rtemp, ExternalAddress((address)safepoint_table));
 618     indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
 619   }
 620 
 621   nop(); // to avoid filling CPU pipeline with invalid instructions
 622   nop();
 623 }
 624 
 625 void InterpreterMacroAssembler::dispatch_only(TosState state, bool generate_poll) {
 626   dispatch_base(state, DispatchDefault, true, generate_poll);
 627 }
 628 
 629 
 630 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) {
 631   dispatch_base(state, DispatchNormal);
 632 }
 633 
 634 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
 635   dispatch_base(state, DispatchNormal, false);
 636 }
 637 
 638 void InterpreterMacroAssembler::dispatch_next(TosState state, int step, bool generate_poll) {
 639   // load next bytecode and advance Rbcp
 640   ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed));
 641   dispatch_base(state, DispatchDefault, true, generate_poll);
 642 }
 643 
 644 void InterpreterMacroAssembler::narrow(Register result) {
 645   // mask integer result to narrower return type.
 646   const Register Rtmp = R2;
 647 
 648   // get method type
 649   ldr(Rtmp, Address(Rmethod, Method::const_offset()));
 650   ldrb(Rtmp, Address(Rtmp, ConstMethod::result_type_offset()));
 651 
 652   Label notBool, notByte, notChar, done;
 653   cmp(Rtmp, T_INT);
 654   b(done, eq);
 655 
 656   cmp(Rtmp, T_BOOLEAN);
 657   b(notBool, ne);
 658   and_32(result, result, 1);
 659   b(done);
 660 
 661   bind(notBool);
 662   cmp(Rtmp, T_BYTE);
 663   b(notByte, ne);
 664   sign_extend(result, result, 8);
 665   b(done);
 666 
 667   bind(notByte);
 668   cmp(Rtmp, T_CHAR);
 669   b(notChar, ne);
 670   zero_extend(result, result, 16);
 671   b(done);
 672 
 673   bind(notChar);
 674   // cmp(Rtmp, T_SHORT);
 675   // b(done, ne);
 676   sign_extend(result, result, 16);
 677 
 678   // Nothing to do
 679   bind(done);
 680 }
 681 
 682 // remove activation
 683 //
 684 // Unlock the receiver if this is a synchronized method.
 685 // Unlock any Java monitors from synchronized blocks.
 686 // Remove the activation from the stack.
 687 //
 688 // If there are locked Java monitors
 689 //    If throw_monitor_exception
 690 //       throws IllegalMonitorStateException
 691 //    Else if install_monitor_exception
 692 //       installs IllegalMonitorStateException
 693 //    Else
 694 //       no error processing
 695 void InterpreterMacroAssembler::remove_activation(TosState state, Register ret_addr,
 696                                                   bool throw_monitor_exception,
 697                                                   bool install_monitor_exception,
 698                                                   bool notify_jvmdi) {
 699   Label unlock, unlocked, no_unlock;
 700 
 701   // Note: Registers R0, R1, S0 and D0 (TOS cached value) may be in use for the result.
 702 
 703   const Address do_not_unlock_if_synchronized(Rthread,
 704                          JavaThread::do_not_unlock_if_synchronized_offset());
 705 
 706   const Register Rflag = R2;
 707   const Register Raccess_flags = R3;
 708 
 709   restore_method();
 710 
 711   ldrb(Rflag, do_not_unlock_if_synchronized);
 712 
 713   // get method access flags
 714   ldr_u32(Raccess_flags, Address(Rmethod, Method::access_flags_offset()));
 715 
 716   strb(zero_register(Rtemp), do_not_unlock_if_synchronized); // reset the flag
 717 
 718   // check if method is synchronized
 719 
 720   tbz(Raccess_flags, JVM_ACC_SYNCHRONIZED_BIT, unlocked);
 721 
 722   // Don't unlock anything if the _do_not_unlock_if_synchronized flag is set.
 723   cbnz(Rflag, no_unlock);
 724 
 725   // unlock monitor
 726   push(state);                                   // save result
 727 
 728   // BasicObjectLock will be first in list, since this is a synchronized method. However, need
 729   // to check that the object has not been unlocked by an explicit monitorexit bytecode.
 730 
 731   const Register Rmonitor = R0;                  // fixed in unlock_object()
 732   const Register Robj = R2;
 733 
 734   // address of first monitor
 735   sub(Rmonitor, FP, - frame::interpreter_frame_monitor_block_bottom_offset * wordSize + (int)sizeof(BasicObjectLock));
 736 
 737   ldr(Robj, Address(Rmonitor, BasicObjectLock::obj_offset_in_bytes()));
 738   cbnz(Robj, unlock);
 739 
 740   pop(state);
 741 
 742   if (throw_monitor_exception) {
 743     // Entry already unlocked, need to throw exception
 744     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
 745     should_not_reach_here();
 746   } else {
 747     // Monitor already unlocked during a stack unroll.
 748     // If requested, install an illegal_monitor_state_exception.
 749     // Continue with stack unrolling.
 750     if (install_monitor_exception) {
 751       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
 752     }
 753     b(unlocked);
 754   }
 755 
 756 
 757   // Exception case for the check that all monitors are unlocked.
 758   const Register Rcur = R2;
 759   Label restart_check_monitors_unlocked, exception_monitor_is_still_locked;
 760 
 761   bind(exception_monitor_is_still_locked);
 762   // Monitor entry is still locked, need to throw exception.
 763   // Rcur: monitor entry.
 764 
 765   if (throw_monitor_exception) {
 766     // Throw exception
 767     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
 768     should_not_reach_here();
 769   } else {
 770     // Stack unrolling. Unlock object and install illegal_monitor_exception
 771     // Unlock does not block, so don't have to worry about the frame
 772 
 773     push(state);
 774     mov(Rmonitor, Rcur);
 775     unlock_object(Rmonitor);
 776 
 777     if (install_monitor_exception) {
 778       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
 779     }
 780 
 781     pop(state);
 782     b(restart_check_monitors_unlocked);
 783   }
 784 
 785   bind(unlock);
 786   unlock_object(Rmonitor);
 787   pop(state);
 788 
 789   // Check that for block-structured locking (i.e., that all locked objects has been unlocked)
 790   bind(unlocked);
 791 
 792   // Check that all monitors are unlocked
 793   {
 794     Label loop;
 795 
 796     const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 797     const Register Rbottom = R3;
 798     const Register Rcur_obj = Rtemp;
 799 
 800     bind(restart_check_monitors_unlocked);
 801 
 802     ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
 803                                  // points to current entry, starting with top-most entry
 804     sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize);
 805                                  // points to word before bottom of monitor block
 806 
 807     cmp(Rcur, Rbottom);          // check if there are no monitors
 808     ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
 809                                  // prefetch monitor's object
 810     b(no_unlock, eq);
 811 
 812     bind(loop);
 813     // check if current entry is used
 814     cbnz(Rcur_obj, exception_monitor_is_still_locked);
 815 
 816     add(Rcur, Rcur, entry_size);      // otherwise advance to next entry
 817     cmp(Rcur, Rbottom);               // check if bottom reached
 818     ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
 819                                       // prefetch monitor's object
 820     b(loop, ne);                      // if not at bottom then check this entry
 821   }
 822 
 823   bind(no_unlock);
 824 
 825   // jvmti support
 826   if (notify_jvmdi) {
 827     notify_method_exit(state, NotifyJVMTI);     // preserve TOSCA
 828   } else {
 829     notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
 830   }
 831 
 832   // remove activation
 833   mov(Rtemp, FP);
 834   ldmia(FP, RegisterSet(FP) | RegisterSet(LR));
 835   ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize));
 836 
 837   if (ret_addr != LR) {
 838     mov(ret_addr, LR);
 839   }
 840 }
 841 
 842 
 843 // At certain points in the method invocation the monitor of
 844 // synchronized methods hasn't been entered yet.
 845 // To correctly handle exceptions at these points, we set the thread local
 846 // variable _do_not_unlock_if_synchronized to true. The remove_activation will
 847 // check this flag.
 848 void InterpreterMacroAssembler::set_do_not_unlock_if_synchronized(bool flag, Register tmp) {
 849   const Address do_not_unlock_if_synchronized(Rthread,
 850                          JavaThread::do_not_unlock_if_synchronized_offset());
 851   if (flag) {
 852     mov(tmp, 1);
 853     strb(tmp, do_not_unlock_if_synchronized);
 854   } else {
 855     strb(zero_register(tmp), do_not_unlock_if_synchronized);
 856   }
 857 }
 858 
 859 // Lock object
 860 //
 861 // Argument: R1 : Points to BasicObjectLock to be used for locking.
 862 // Must be initialized with object to lock.
 863 // Blows volatile registers R0-R3, Rtemp, LR. Calls VM.
 864 void InterpreterMacroAssembler::lock_object(Register Rlock) {
 865   assert(Rlock == R1, "the second argument");
 866   const Register Robj = R2;
 867   assert_different_registers(Robj, Rlock);
 868   const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
 869 
 870   // Load object pointer
 871   ldr(Robj, Address(Rlock, obj_offset));
 872 
 873   // TODO: Implement fast-locking.
 874   mov(R0, Robj);
 875   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), R0);
 876 }
 877 
 878 
 879 // Unlocks an object. Used in monitorexit bytecode and remove_activation.
 880 //
 881 // Argument: R0: Points to BasicObjectLock structure for lock
 882 // Throw an IllegalMonitorException if object is not locked by current thread
 883 // Blows volatile registers R0-R3, Rtemp, LR. Calls VM.
 884 void InterpreterMacroAssembler::unlock_object(Register Rlock) {
 885   assert(Rlock == R0, "the first argument");
 886   const Register Robj = R2;
 887   assert_different_registers(Robj, Rlock);
 888   const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
 889 
 890   // Load oop into Robj
 891   ldr(Robj, Address(Rlock, obj_offset));
 892 
 893   // TODO: Implement fast-locking.
 894   mov(R0, Robj);
 895   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), R0);
 896 }
 897 
 898 
 899 // Test ImethodDataPtr.  If it is null, continue at the specified label
 900 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) {
 901   assert(ProfileInterpreter, "must be profiling interpreter");
 902   ldr(mdp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
 903   cbz(mdp, zero_continue);
 904 }
 905 
 906 
 907 // Set the method data pointer for the current bcp.
 908 // Blows volatile registers R0-R3, Rtemp, LR.
 909 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
 910   assert(ProfileInterpreter, "must be profiling interpreter");
 911   Label set_mdp;
 912 
 913   // Test MDO to avoid the call if it is NULL.
 914   ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
 915   cbz(Rtemp, set_mdp);
 916 
 917   mov(R0, Rmethod);
 918   mov(R1, Rbcp);
 919   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), R0, R1);
 920   // R0/W0: mdi
 921 
 922   // mdo is guaranteed to be non-zero here, we checked for it before the call.
 923   ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
 924   add(Rtemp, Rtemp, in_bytes(MethodData::data_offset()));
 925   add_ptr_scaled_int32(Rtemp, Rtemp, R0, 0);
 926 
 927   bind(set_mdp);
 928   str(Rtemp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
 929 }
 930 
 931 
 932 void InterpreterMacroAssembler::verify_method_data_pointer() {
 933   assert(ProfileInterpreter, "must be profiling interpreter");
 934 #ifdef ASSERT
 935   Label verify_continue;
 936   save_caller_save_registers();
 937 
 938   const Register Rmdp = R2;
 939   test_method_data_pointer(Rmdp, verify_continue); // If mdp is zero, continue
 940 
 941   // If the mdp is valid, it will point to a DataLayout header which is
 942   // consistent with the bcp.  The converse is highly probable also.
 943 
 944   ldrh(R3, Address(Rmdp, DataLayout::bci_offset()));
 945   ldr(Rtemp, Address(Rmethod, Method::const_offset()));
 946   add(R3, R3, Rtemp);
 947   add(R3, R3, in_bytes(ConstMethod::codes_offset()));
 948   cmp(R3, Rbcp);
 949   b(verify_continue, eq);
 950 
 951   mov(R0, Rmethod);
 952   mov(R1, Rbcp);
 953   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R0, R1, Rmdp);
 954 
 955   bind(verify_continue);
 956   restore_caller_save_registers();
 957 #endif // ASSERT
 958 }
 959 
 960 
 961 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, int offset, Register value) {
 962   assert(ProfileInterpreter, "must be profiling interpreter");
 963   assert_different_registers(mdp_in, value);
 964   str(value, Address(mdp_in, offset));
 965 }
 966 
 967 
 968 // Increments mdp data. Sets bumped_count register to adjusted counter.
 969 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
 970                                                       int offset,
 971                                                       Register bumped_count,
 972                                                       bool decrement) {
 973   assert(ProfileInterpreter, "must be profiling interpreter");
 974 
 975   // Counter address
 976   Address data(mdp_in, offset);
 977   assert_different_registers(mdp_in, bumped_count);
 978 
 979   increment_mdp_data_at(data, bumped_count, decrement);
 980 }
 981 
 982 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, int flag_byte_constant) {
 983   assert_different_registers(mdp_in, Rtemp);
 984   assert(ProfileInterpreter, "must be profiling interpreter");
 985   assert((0 < flag_byte_constant) && (flag_byte_constant < (1 << BitsPerByte)), "flag mask is out of range");
 986 
 987   // Set the flag
 988   ldrb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset())));
 989   orr(Rtemp, Rtemp, (unsigned)flag_byte_constant);
 990   strb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset())));
 991 }
 992 
 993 
 994 // Increments mdp data. Sets bumped_count register to adjusted counter.
 995 void InterpreterMacroAssembler::increment_mdp_data_at(Address data,
 996                                                       Register bumped_count,
 997                                                       bool decrement) {
 998   assert(ProfileInterpreter, "must be profiling interpreter");
 999 
1000   ldr(bumped_count, data);
1001   if (decrement) {
1002     // Decrement the register. Set condition codes.
1003     subs(bumped_count, bumped_count, DataLayout::counter_increment);
1004     // Avoid overflow.
1005     add(bumped_count, bumped_count, DataLayout::counter_increment, pl);
1006   } else {
1007     // Increment the register. Set condition codes.
1008     adds(bumped_count, bumped_count, DataLayout::counter_increment);
1009     // Avoid overflow.
1010     sub(bumped_count, bumped_count, DataLayout::counter_increment, mi);
1011   }
1012   str(bumped_count, data);
1013 }
1014 
1015 
1016 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in,
1017                                                  int offset,
1018                                                  Register value,
1019                                                  Register test_value_out,
1020                                                  Label& not_equal_continue) {
1021   assert(ProfileInterpreter, "must be profiling interpreter");
1022   assert_different_registers(mdp_in, test_value_out, value);
1023 
1024   ldr(test_value_out, Address(mdp_in, offset));
1025   cmp(test_value_out, value);
1026 
1027   b(not_equal_continue, ne);
1028 }
1029 
1030 
1031 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, int offset_of_disp, Register reg_temp) {
1032   assert(ProfileInterpreter, "must be profiling interpreter");
1033   assert_different_registers(mdp_in, reg_temp);
1034 
1035   ldr(reg_temp, Address(mdp_in, offset_of_disp));
1036   add(mdp_in, mdp_in, reg_temp);
1037   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1038 }
1039 
1040 
1041 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, Register reg_offset, Register reg_tmp) {
1042   assert(ProfileInterpreter, "must be profiling interpreter");
1043   assert_different_registers(mdp_in, reg_offset, reg_tmp);
1044 
1045   ldr(reg_tmp, Address(mdp_in, reg_offset));
1046   add(mdp_in, mdp_in, reg_tmp);
1047   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1048 }
1049 
1050 
1051 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, int constant) {
1052   assert(ProfileInterpreter, "must be profiling interpreter");
1053   add(mdp_in, mdp_in, constant);
1054   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1055 }
1056 
1057 
1058 // Blows volatile registers R0-R3, Rtemp, LR).
1059 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {
1060   assert(ProfileInterpreter, "must be profiling interpreter");
1061   assert_different_registers(return_bci, R0, R1, R2, R3, Rtemp);
1062 
1063   mov(R1, return_bci);
1064   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), R1);
1065 }
1066 
1067 
1068 // Sets mdp, bumped_count registers, blows Rtemp.
1069 void InterpreterMacroAssembler::profile_taken_branch(Register mdp, Register bumped_count) {
1070   assert_different_registers(mdp, bumped_count);
1071 
1072   if (ProfileInterpreter) {
1073     Label profile_continue;
1074 
1075     // If no method data exists, go to profile_continue.
1076     // Otherwise, assign to mdp
1077     test_method_data_pointer(mdp, profile_continue);
1078 
1079     // We are taking a branch. Increment the taken count.
1080     increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()), bumped_count);
1081 
1082     // The method data pointer needs to be updated to reflect the new target.
1083     update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()), Rtemp);
1084 
1085     bind (profile_continue);
1086   }
1087 }
1088 
1089 
1090 // Sets mdp, blows Rtemp.
1091 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
1092   assert_different_registers(mdp, Rtemp);
1093 
1094   if (ProfileInterpreter) {
1095     Label profile_continue;
1096 
1097     // If no method data exists, go to profile_continue.
1098     test_method_data_pointer(mdp, profile_continue);
1099 
1100     // We are taking a branch.  Increment the not taken count.
1101     increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()), Rtemp);
1102 
1103     // The method data pointer needs to be updated to correspond to the next bytecode
1104     update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
1105 
1106     bind (profile_continue);
1107   }
1108 }
1109 
1110 
1111 // Sets mdp, blows Rtemp.
1112 void InterpreterMacroAssembler::profile_call(Register mdp) {
1113   assert_different_registers(mdp, Rtemp);
1114 
1115   if (ProfileInterpreter) {
1116     Label profile_continue;
1117 
1118     // If no method data exists, go to profile_continue.
1119     test_method_data_pointer(mdp, profile_continue);
1120 
1121     // We are making a call.  Increment the count.
1122     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1123 
1124     // The method data pointer needs to be updated to reflect the new target.
1125     update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1126 
1127     bind (profile_continue);
1128   }
1129 }
1130 
1131 
1132 // Sets mdp, blows Rtemp.
1133 void InterpreterMacroAssembler::profile_final_call(Register mdp) {
1134   if (ProfileInterpreter) {
1135     Label profile_continue;
1136 
1137     // If no method data exists, go to profile_continue.
1138     test_method_data_pointer(mdp, profile_continue);
1139 
1140     // We are making a call.  Increment the count.
1141     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1142 
1143     // The method data pointer needs to be updated to reflect the new target.
1144     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1145 
1146     bind (profile_continue);
1147   }
1148 }
1149 
1150 
1151 // Sets mdp, blows Rtemp.
1152 void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) {
1153   assert_different_registers(mdp, receiver, Rtemp);
1154 
1155   if (ProfileInterpreter) {
1156     Label profile_continue;
1157 
1158     // If no method data exists, go to profile_continue.
1159     test_method_data_pointer(mdp, profile_continue);
1160 
1161     Label skip_receiver_profile;
1162     if (receiver_can_be_null) {
1163       Label not_null;
1164       cbnz(receiver, not_null);
1165       // We are making a call.  Increment the count for null receiver.
1166       increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1167       b(skip_receiver_profile);
1168       bind(not_null);
1169     }
1170 
1171     // Record the receiver type.
1172     record_klass_in_profile(receiver, mdp, Rtemp, true);
1173     bind(skip_receiver_profile);
1174 
1175     // The method data pointer needs to be updated to reflect the new target.
1176     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1177     bind(profile_continue);
1178   }
1179 }
1180 
1181 
1182 void InterpreterMacroAssembler::record_klass_in_profile_helper(
1183                                         Register receiver, Register mdp,
1184                                         Register reg_tmp,
1185                                         int start_row, Label& done, bool is_virtual_call) {
1186   if (TypeProfileWidth == 0)
1187     return;
1188 
1189   assert_different_registers(receiver, mdp, reg_tmp);
1190 
1191   int last_row = VirtualCallData::row_limit() - 1;
1192   assert(start_row <= last_row, "must be work left to do");
1193   // Test this row for both the receiver and for null.
1194   // Take any of three different outcomes:
1195   //   1. found receiver => increment count and goto done
1196   //   2. found null => keep looking for case 1, maybe allocate this cell
1197   //   3. found something else => keep looking for cases 1 and 2
1198   // Case 3 is handled by a recursive call.
1199   for (int row = start_row; row <= last_row; row++) {
1200     Label next_test;
1201 
1202     // See if the receiver is receiver[n].
1203     int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row));
1204 
1205     test_mdp_data_at(mdp, recvr_offset, receiver, reg_tmp, next_test);
1206 
1207     // The receiver is receiver[n].  Increment count[n].
1208     int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row));
1209     increment_mdp_data_at(mdp, count_offset, reg_tmp);
1210     b(done);
1211 
1212     bind(next_test);
1213     // reg_tmp now contains the receiver from the CallData.
1214 
1215     if (row == start_row) {
1216       Label found_null;
1217       // Failed the equality check on receiver[n]...  Test for null.
1218       if (start_row == last_row) {
1219         // The only thing left to do is handle the null case.
1220         if (is_virtual_call) {
1221           cbz(reg_tmp, found_null);
1222           // Receiver did not match any saved receiver and there is no empty row for it.
1223           // Increment total counter to indicate polymorphic case.
1224           increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), reg_tmp);
1225           b(done);
1226           bind(found_null);
1227         } else {
1228           cbnz(reg_tmp, done);
1229         }
1230         break;
1231       }
1232       // Since null is rare, make it be the branch-taken case.
1233       cbz(reg_tmp, found_null);
1234 
1235       // Put all the "Case 3" tests here.
1236       record_klass_in_profile_helper(receiver, mdp, reg_tmp, start_row + 1, done, is_virtual_call);
1237 
1238       // Found a null.  Keep searching for a matching receiver,
1239       // but remember that this is an empty (unused) slot.
1240       bind(found_null);
1241     }
1242   }
1243 
1244   // In the fall-through case, we found no matching receiver, but we
1245   // observed the receiver[start_row] is NULL.
1246 
1247   // Fill in the receiver field and increment the count.
1248   int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row));
1249   set_mdp_data_at(mdp, recvr_offset, receiver);
1250   int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row));
1251   mov(reg_tmp, DataLayout::counter_increment);
1252   set_mdp_data_at(mdp, count_offset, reg_tmp);
1253   if (start_row > 0) {
1254     b(done);
1255   }
1256 }
1257 
1258 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
1259                                                         Register mdp,
1260                                                         Register reg_tmp,
1261                                                         bool is_virtual_call) {
1262   assert(ProfileInterpreter, "must be profiling");
1263   assert_different_registers(receiver, mdp, reg_tmp);
1264 
1265   Label done;
1266 
1267   record_klass_in_profile_helper(receiver, mdp, reg_tmp, 0, done, is_virtual_call);
1268 
1269   bind (done);
1270 }
1271 
1272 // Sets mdp, blows volatile registers R0-R3, Rtemp, LR).
1273 void InterpreterMacroAssembler::profile_ret(Register mdp, Register return_bci) {
1274   assert_different_registers(mdp, return_bci, Rtemp, R0, R1, R2, R3);
1275 
1276   if (ProfileInterpreter) {
1277     Label profile_continue;
1278     uint row;
1279 
1280     // If no method data exists, go to profile_continue.
1281     test_method_data_pointer(mdp, profile_continue);
1282 
1283     // Update the total ret count.
1284     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1285 
1286     for (row = 0; row < RetData::row_limit(); row++) {
1287       Label next_test;
1288 
1289       // See if return_bci is equal to bci[n]:
1290       test_mdp_data_at(mdp, in_bytes(RetData::bci_offset(row)), return_bci,
1291                        Rtemp, next_test);
1292 
1293       // return_bci is equal to bci[n].  Increment the count.
1294       increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)), Rtemp);
1295 
1296       // The method data pointer needs to be updated to reflect the new target.
1297       update_mdp_by_offset(mdp, in_bytes(RetData::bci_displacement_offset(row)), Rtemp);
1298       b(profile_continue);
1299       bind(next_test);
1300     }
1301 
1302     update_mdp_for_ret(return_bci);
1303 
1304     bind(profile_continue);
1305   }
1306 }
1307 
1308 
1309 // Sets mdp.
1310 void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
1311   if (ProfileInterpreter) {
1312     Label profile_continue;
1313 
1314     // If no method data exists, go to profile_continue.
1315     test_method_data_pointer(mdp, profile_continue);
1316 
1317     set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1318 
1319     // The method data pointer needs to be updated.
1320     int mdp_delta = in_bytes(BitData::bit_data_size());
1321     if (TypeProfileCasts) {
1322       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1323     }
1324     update_mdp_by_constant(mdp, mdp_delta);
1325 
1326     bind (profile_continue);
1327   }
1328 }
1329 
1330 
1331 // Sets mdp, blows Rtemp.
1332 void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) {
1333   assert_different_registers(mdp, Rtemp);
1334 
1335   if (ProfileInterpreter && TypeProfileCasts) {
1336     Label profile_continue;
1337 
1338     // If no method data exists, go to profile_continue.
1339     test_method_data_pointer(mdp, profile_continue);
1340 
1341     int count_offset = in_bytes(CounterData::count_offset());
1342     // Back up the address, since we have already bumped the mdp.
1343     count_offset -= in_bytes(VirtualCallData::virtual_call_data_size());
1344 
1345     // *Decrement* the counter.  We expect to see zero or small negatives.
1346     increment_mdp_data_at(mdp, count_offset, Rtemp, true);
1347 
1348     bind (profile_continue);
1349   }
1350 }
1351 
1352 
1353 // Sets mdp, blows Rtemp.
1354 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass)
1355 {
1356   assert_different_registers(mdp, klass, Rtemp);
1357 
1358   if (ProfileInterpreter) {
1359     Label profile_continue;
1360 
1361     // If no method data exists, go to profile_continue.
1362     test_method_data_pointer(mdp, profile_continue);
1363 
1364     // The method data pointer needs to be updated.
1365     int mdp_delta = in_bytes(BitData::bit_data_size());
1366     if (TypeProfileCasts) {
1367       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1368 
1369       // Record the object type.
1370       record_klass_in_profile(klass, mdp, Rtemp, false);
1371     }
1372     update_mdp_by_constant(mdp, mdp_delta);
1373 
1374     bind(profile_continue);
1375   }
1376 }
1377 
1378 
1379 // Sets mdp, blows Rtemp.
1380 void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
1381   assert_different_registers(mdp, Rtemp);
1382 
1383   if (ProfileInterpreter) {
1384     Label profile_continue;
1385 
1386     // If no method data exists, go to profile_continue.
1387     test_method_data_pointer(mdp, profile_continue);
1388 
1389     // Update the default case count
1390     increment_mdp_data_at(mdp, in_bytes(MultiBranchData::default_count_offset()), Rtemp);
1391 
1392     // The method data pointer needs to be updated.
1393     update_mdp_by_offset(mdp, in_bytes(MultiBranchData::default_displacement_offset()), Rtemp);
1394 
1395     bind(profile_continue);
1396   }
1397 }
1398 
1399 
1400 // Sets mdp. Blows reg_tmp1, reg_tmp2. Index could be the same as reg_tmp2.
1401 void InterpreterMacroAssembler::profile_switch_case(Register mdp, Register index, Register reg_tmp1, Register reg_tmp2) {
1402   assert_different_registers(mdp, reg_tmp1, reg_tmp2);
1403   assert_different_registers(mdp, reg_tmp1, index);
1404 
1405   if (ProfileInterpreter) {
1406     Label profile_continue;
1407 
1408     const int count_offset = in_bytes(MultiBranchData::case_array_offset()) +
1409                               in_bytes(MultiBranchData::relative_count_offset());
1410 
1411     const int displacement_offset = in_bytes(MultiBranchData::case_array_offset()) +
1412                               in_bytes(MultiBranchData::relative_displacement_offset());
1413 
1414     // If no method data exists, go to profile_continue.
1415     test_method_data_pointer(mdp, profile_continue);
1416 
1417     // Build the base (index * per_case_size_in_bytes())
1418     logical_shift_left(reg_tmp1, index, exact_log2(in_bytes(MultiBranchData::per_case_size())));
1419 
1420     // Update the case count
1421     add(reg_tmp1, reg_tmp1, count_offset);
1422     increment_mdp_data_at(Address(mdp, reg_tmp1), reg_tmp2);
1423 
1424     // The method data pointer needs to be updated.
1425     add(reg_tmp1, reg_tmp1, displacement_offset - count_offset);
1426     update_mdp_by_offset(mdp, reg_tmp1, reg_tmp2);
1427 
1428     bind (profile_continue);
1429   }
1430 }
1431 
1432 
1433 void InterpreterMacroAssembler::byteswap_u32(Register r, Register rtmp1, Register rtmp2) {
1434   if (VM_Version::supports_rev()) {
1435     rev(r, r);
1436   } else {
1437     eor(rtmp1, r, AsmOperand(r, ror, 16));
1438     mvn(rtmp2, 0x0000ff00);
1439     andr(rtmp1, rtmp2, AsmOperand(rtmp1, lsr, 8));
1440     eor(r, rtmp1, AsmOperand(r, ror, 8));
1441   }
1442 }
1443 
1444 
1445 void InterpreterMacroAssembler::inc_global_counter(address address_of_counter, int offset, Register tmp1, Register tmp2, bool avoid_overflow) {
1446   const intx addr = (intx) (address_of_counter + offset);
1447 
1448   assert ((addr & 0x3) == 0, "address of counter should be aligned");
1449   const intx offset_mask = right_n_bits(12);
1450 
1451   const address base = (address) (addr & ~offset_mask);
1452   const int offs = (int) (addr & offset_mask);
1453 
1454   const Register addr_base = tmp1;
1455   const Register val = tmp2;
1456 
1457   mov_slow(addr_base, base);
1458   ldr_s32(val, Address(addr_base, offs));
1459 
1460   if (avoid_overflow) {
1461     adds_32(val, val, 1);
1462     str(val, Address(addr_base, offs), pl);
1463   } else {
1464     add_32(val, val, 1);
1465     str_32(val, Address(addr_base, offs));
1466   }
1467 }
1468 
1469 void InterpreterMacroAssembler::interp_verify_oop(Register reg, TosState state, const char *file, int line) {
1470   if (state == atos) { MacroAssembler::_verify_oop(reg, "broken oop", file, line); }
1471 }
1472 
1473 // Inline assembly for:
1474 //
1475 // if (thread is in interp_only_mode) {
1476 //   InterpreterRuntime::post_method_entry();
1477 // }
1478 // if (DTraceMethodProbes) {
1479 //   SharedRuntime::dtrace_method_entry(method, receiver);
1480 // }
1481 // if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) {
1482 //   SharedRuntime::rc_trace_method_entry(method, receiver);
1483 // }
1484 
1485 void InterpreterMacroAssembler::notify_method_entry() {
1486   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1487   // track stack depth.  If it is possible to enter interp_only_mode we add
1488   // the code to check if the event should be sent.
1489   if (can_post_interpreter_events()) {
1490     Label L;
1491 
1492     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
1493     cbz(Rtemp, L);
1494 
1495     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry));
1496 
1497     bind(L);
1498   }
1499 
1500   // Note: Disable DTrace runtime check for now to eliminate overhead on each method entry
1501   if (DTraceMethodProbes) {
1502     Label Lcontinue;
1503 
1504     ldrb_global(Rtemp, (address)&DTraceMethodProbes);
1505     cbz(Rtemp, Lcontinue);
1506 
1507     mov(R0, Rthread);
1508     mov(R1, Rmethod);
1509     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), R0, R1);
1510 
1511     bind(Lcontinue);
1512   }
1513   // RedefineClasses() tracing support for obsolete method entry
1514   if (log_is_enabled(Trace, redefine, class, obsolete)) {
1515     mov(R0, Rthread);
1516     mov(R1, Rmethod);
1517     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
1518                  R0, R1);
1519   }
1520 }
1521 
1522 
1523 void InterpreterMacroAssembler::notify_method_exit(
1524                  TosState state, NotifyMethodExitMode mode,
1525                  bool native, Register result_lo, Register result_hi, FloatRegister result_fp) {
1526   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1527   // track stack depth.  If it is possible to enter interp_only_mode we add
1528   // the code to check if the event should be sent.
1529   if (mode == NotifyJVMTI && can_post_interpreter_events()) {
1530     Label L;
1531     // Note: frame::interpreter_frame_result has a dependency on how the
1532     // method result is saved across the call to post_method_exit. If this
1533     // is changed then the interpreter_frame_result implementation will
1534     // need to be updated too.
1535 
1536     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
1537     cbz(Rtemp, L);
1538 
1539     if (native) {
1540       // For c++ and template interpreter push both result registers on the
1541       // stack in native, we don't know the state.
1542       // See frame::interpreter_frame_result for code that gets the result values from here.
1543       assert(result_lo != noreg, "result registers should be defined");
1544 
1545       assert(result_hi != noreg, "result registers should be defined");
1546 
1547 #ifdef __ABI_HARD__
1548       assert(result_fp != fnoreg, "FP result register must be defined");
1549       sub(SP, SP, 2 * wordSize);
1550       fstd(result_fp, Address(SP));
1551 #endif // __ABI_HARD__
1552 
1553       push(RegisterSet(result_lo) | RegisterSet(result_hi));
1554 
1555       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
1556 
1557       pop(RegisterSet(result_lo) | RegisterSet(result_hi));
1558 #ifdef __ABI_HARD__
1559       fldd(result_fp, Address(SP));
1560       add(SP, SP, 2 * wordSize);
1561 #endif // __ABI_HARD__
1562 
1563     } else {
1564       // For the template interpreter, the value on tos is the size of the
1565       // state. (c++ interpreter calls jvmti somewhere else).
1566       push(state);
1567       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
1568       pop(state);
1569     }
1570 
1571     bind(L);
1572   }
1573 
1574   // Note: Disable DTrace runtime check for now to eliminate overhead on each method exit
1575   if (DTraceMethodProbes) {
1576     Label Lcontinue;
1577 
1578     ldrb_global(Rtemp, (address)&DTraceMethodProbes);
1579     cbz(Rtemp, Lcontinue);
1580 
1581     push(state);
1582 
1583     mov(R0, Rthread);
1584     mov(R1, Rmethod);
1585 
1586     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), R0, R1);
1587 
1588     pop(state);
1589 
1590     bind(Lcontinue);
1591   }
1592 }
1593 
1594 
1595 #ifndef PRODUCT
1596 
1597 void InterpreterMacroAssembler::trace_state(const char* msg) {
1598   int push_size = save_caller_save_registers();
1599 
1600   Label Lcontinue;
1601   InlinedString Lmsg0("%s: FP=" INTPTR_FORMAT ", SP=" INTPTR_FORMAT "\n");
1602   InlinedString Lmsg(msg);
1603   InlinedAddress Lprintf((address)printf);
1604 
1605   ldr_literal(R0, Lmsg0);
1606   ldr_literal(R1, Lmsg);
1607   mov(R2, FP);
1608   add(R3, SP, push_size);  // original SP (without saved registers)
1609   ldr_literal(Rtemp, Lprintf);
1610   call(Rtemp);
1611 
1612   b(Lcontinue);
1613 
1614   bind_literal(Lmsg0);
1615   bind_literal(Lmsg);
1616   bind_literal(Lprintf);
1617 
1618 
1619   bind(Lcontinue);
1620 
1621   restore_caller_save_registers();
1622 }
1623 
1624 #endif
1625 
1626 // Jump if ((*counter_addr += increment) & mask) satisfies the condition.
1627 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
1628                                                         int increment, Address mask_addr,
1629                                                         Register scratch, Register scratch2,
1630                                                         AsmCondition cond, Label* where) {
1631   // caution: scratch2 and base address of counter_addr can be the same
1632   assert_different_registers(scratch, scratch2);
1633   ldr_u32(scratch, counter_addr);
1634   add(scratch, scratch, increment);
1635   str_32(scratch, counter_addr);
1636 
1637   ldr(scratch2, mask_addr);
1638   andrs(scratch, scratch, scratch2);
1639   b(*where, cond);
1640 }
1641 
1642 void InterpreterMacroAssembler::get_method_counters(Register method,
1643                                                     Register Rcounters,
1644                                                     Label& skip,
1645                                                     bool saveRegs,
1646                                                     Register reg1,
1647                                                     Register reg2,
1648                                                     Register reg3) {
1649   const Address method_counters(method, Method::method_counters_offset());
1650   Label has_counters;
1651 
1652   ldr(Rcounters, method_counters);
1653   cbnz(Rcounters, has_counters);
1654 
1655   if (saveRegs) {
1656     // Save and restore in use caller-saved registers since they will be trashed by call_VM
1657     assert(reg1 != noreg, "must specify reg1");
1658     assert(reg2 != noreg, "must specify reg2");
1659     assert(reg3 == noreg, "must not specify reg3");
1660     push(RegisterSet(reg1) | RegisterSet(reg2));
1661   }
1662 
1663   mov(R1, method);
1664   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::build_method_counters), R1);
1665 
1666   if (saveRegs) {
1667     pop(RegisterSet(reg1) | RegisterSet(reg2));
1668   }
1669 
1670   ldr(Rcounters, method_counters);
1671   cbz(Rcounters, skip); // No MethodCounters created, OutOfMemory
1672 
1673   bind(has_counters);
1674 }