1 /*
   2  * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/cardTable.hpp"
  30 #include "gc/shared/cardTableBarrierSet.inline.hpp"
  31 #include "gc/shared/collectedHeap.hpp"
  32 #include "interp_masm_arm.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "interpreter/interpreterRuntime.hpp"
  35 #include "logging/log.hpp"
  36 #include "oops/arrayOop.hpp"
  37 #include "oops/markWord.hpp"
  38 #include "oops/method.hpp"
  39 #include "oops/methodData.hpp"
  40 #include "prims/jvmtiExport.hpp"
  41 #include "prims/jvmtiThreadState.hpp"
  42 #include "runtime/basicLock.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/safepointMechanism.hpp"
  45 #include "runtime/sharedRuntime.hpp"
  46 #include "utilities/powerOfTwo.hpp"
  47 
  48 //--------------------------------------------------------------------
  49 // Implementation of InterpreterMacroAssembler
  50 
  51 
  52 
  53 
  54 InterpreterMacroAssembler::InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) {
  55 }
  56 
  57 void InterpreterMacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
  58 #ifdef ASSERT
  59   // Ensure that last_sp is not filled.
  60   { Label L;
  61     ldr(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
  62     cbz(Rtemp, L);
  63     stop("InterpreterMacroAssembler::call_VM_helper: last_sp != NULL");
  64     bind(L);
  65   }
  66 #endif // ASSERT
  67 
  68   // Rbcp must be saved/restored since it may change due to GC.
  69   save_bcp();
  70 
  71 
  72   // super call
  73   MacroAssembler::call_VM_helper(oop_result, entry_point, number_of_arguments, check_exceptions);
  74 
  75 
  76   // Restore interpreter specific registers.
  77   restore_bcp();
  78   restore_method();
  79 }
  80 
  81 void InterpreterMacroAssembler::jump_to_entry(address entry) {
  82   assert(entry, "Entry must have been generated by now");
  83   b(entry);
  84 }
  85 
  86 void InterpreterMacroAssembler::check_and_handle_popframe() {
  87   if (can_pop_frame()) {
  88     Label L;
  89     const Register popframe_cond = R2_tmp;
  90 
  91     // Initiate popframe handling only if it is not already being processed.  If the flag
  92     // has the popframe_processing bit set, it means that this code is called *during* popframe
  93     // handling - we don't want to reenter.
  94 
  95     ldr_s32(popframe_cond, Address(Rthread, JavaThread::popframe_condition_offset()));
  96     tbz(popframe_cond, exact_log2(JavaThread::popframe_pending_bit), L);
  97     tbnz(popframe_cond, exact_log2(JavaThread::popframe_processing_bit), L);
  98 
  99     // Call Interpreter::remove_activation_preserving_args_entry() to get the
 100     // address of the same-named entrypoint in the generated interpreter code.
 101     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));
 102 
 103     // Call indirectly to avoid generation ordering problem.
 104     jump(R0);
 105 
 106     bind(L);
 107   }
 108 }
 109 
 110 
 111 // Blows R2, Rtemp. Sets TOS cached value.
 112 void InterpreterMacroAssembler::load_earlyret_value(TosState state) {
 113   const Register thread_state = R2_tmp;
 114 
 115   ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
 116 
 117   const Address tos_addr(thread_state, JvmtiThreadState::earlyret_tos_offset());
 118   const Address oop_addr(thread_state, JvmtiThreadState::earlyret_oop_offset());
 119   const Address val_addr(thread_state, JvmtiThreadState::earlyret_value_offset());
 120   const Address val_addr_hi(thread_state, JvmtiThreadState::earlyret_value_offset()
 121                              + in_ByteSize(wordSize));
 122 
 123   Register zero = zero_register(Rtemp);
 124 
 125   switch (state) {
 126     case atos: ldr(R0_tos, oop_addr);
 127                str(zero, oop_addr);
 128                interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 129                break;
 130 
 131     case ltos: ldr(R1_tos_hi, val_addr_hi);        // fall through
 132     case btos:                                     // fall through
 133     case ztos:                                     // fall through
 134     case ctos:                                     // fall through
 135     case stos:                                     // fall through
 136     case itos: ldr_s32(R0_tos, val_addr);          break;
 137 #ifdef __SOFTFP__
 138     case dtos: ldr(R1_tos_hi, val_addr_hi);        // fall through
 139     case ftos: ldr(R0_tos, val_addr);              break;
 140 #else
 141     case ftos: ldr_float (S0_tos, val_addr);       break;
 142     case dtos: ldr_double(D0_tos, val_addr);       break;
 143 #endif // __SOFTFP__
 144     case vtos: /* nothing to do */                 break;
 145     default  : ShouldNotReachHere();
 146   }
 147   // Clean up tos value in the thread object
 148   str(zero, val_addr);
 149   str(zero, val_addr_hi);
 150 
 151   mov(Rtemp, (int) ilgl);
 152   str_32(Rtemp, tos_addr);
 153 }
 154 
 155 
 156 // Blows R2, Rtemp.
 157 void InterpreterMacroAssembler::check_and_handle_earlyret() {
 158   if (can_force_early_return()) {
 159     Label L;
 160     const Register thread_state = R2_tmp;
 161 
 162     ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
 163     cbz(thread_state, L); // if (thread->jvmti_thread_state() == NULL) exit;
 164 
 165     // Initiate earlyret handling only if it is not already being processed.
 166     // If the flag has the earlyret_processing bit set, it means that this code
 167     // is called *during* earlyret handling - we don't want to reenter.
 168 
 169     ldr_s32(Rtemp, Address(thread_state, JvmtiThreadState::earlyret_state_offset()));
 170     cmp(Rtemp, JvmtiThreadState::earlyret_pending);
 171     b(L, ne);
 172 
 173     // Call Interpreter::remove_activation_early_entry() to get the address of the
 174     // same-named entrypoint in the generated interpreter code.
 175 
 176     ldr_s32(R0, Address(thread_state, JvmtiThreadState::earlyret_tos_offset()));
 177     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), R0);
 178 
 179     jump(R0);
 180 
 181     bind(L);
 182   }
 183 }
 184 
 185 
 186 // Sets reg. Blows Rtemp.
 187 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) {
 188   assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");
 189   assert(reg != Rtemp, "should be different registers");
 190 
 191   ldrb(Rtemp, Address(Rbcp, bcp_offset));
 192   ldrb(reg, Address(Rbcp, bcp_offset+1));
 193   orr(reg, reg, AsmOperand(Rtemp, lsl, BitsPerByte));
 194 }
 195 
 196 void InterpreterMacroAssembler::get_index_at_bcp(Register index, int bcp_offset, Register tmp_reg, size_t index_size) {
 197   assert_different_registers(index, tmp_reg);
 198   if (index_size == sizeof(u2)) {
 199     // load bytes of index separately to avoid unaligned access
 200     ldrb(index, Address(Rbcp, bcp_offset+1));
 201     ldrb(tmp_reg, Address(Rbcp, bcp_offset));
 202     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 203   } else if (index_size == sizeof(u4)) {
 204     ldrb(index, Address(Rbcp, bcp_offset+3));
 205     ldrb(tmp_reg, Address(Rbcp, bcp_offset+2));
 206     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 207     ldrb(tmp_reg, Address(Rbcp, bcp_offset+1));
 208     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 209     ldrb(tmp_reg, Address(Rbcp, bcp_offset));
 210     orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
 211     // Check if the secondary index definition is still ~x, otherwise
 212     // we have to change the following assembler code to calculate the
 213     // plain index.
 214     assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line");
 215     mvn_32(index, index);  // convert to plain index
 216   } else if (index_size == sizeof(u1)) {
 217     ldrb(index, Address(Rbcp, bcp_offset));
 218   } else {
 219     ShouldNotReachHere();
 220   }
 221 }
 222 
 223 // Sets cache, index.
 224 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size) {
 225   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 226   assert_different_registers(cache, index);
 227 
 228   get_index_at_bcp(index, bcp_offset, cache, index_size);
 229 
 230   // load constant pool cache pointer
 231   ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));
 232 
 233   // convert from field index to ConstantPoolCacheEntry index
 234   assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
 235   logical_shift_left(index, index, 2);
 236 }
 237 
 238 // Sets cache, index, bytecode.
 239 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size) {
 240   get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size);
 241   // caution index and bytecode can be the same
 242   add(bytecode, cache, AsmOperand(index, lsl, LogBytesPerWord));
 243   ldrb(bytecode, Address(bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset())));
 244   TemplateTable::volatile_barrier(MacroAssembler::LoadLoad, noreg, true);
 245 }
 246 
 247 // Sets cache. Blows reg_tmp.
 248 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register reg_tmp, int bcp_offset, size_t index_size) {
 249   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 250   assert_different_registers(cache, reg_tmp);
 251 
 252   get_index_at_bcp(reg_tmp, bcp_offset, cache, index_size);
 253 
 254   // load constant pool cache pointer
 255   ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));
 256 
 257   // skip past the header
 258   add(cache, cache, in_bytes(ConstantPoolCache::base_offset()));
 259   // convert from field index to ConstantPoolCacheEntry index
 260   // and from word offset to byte offset
 261   assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
 262   add(cache, cache, AsmOperand(reg_tmp, lsl, 2 + LogBytesPerWord));
 263 }
 264 
 265 // Load object from cpool->resolved_references(index)
 266 void InterpreterMacroAssembler::load_resolved_reference_at_index(
 267                                            Register result, Register index) {
 268   assert_different_registers(result, index);
 269   get_constant_pool(result);
 270 
 271   Register cache = result;
 272   // load pointer for resolved_references[] objArray
 273   ldr(cache, Address(result, ConstantPool::cache_offset_in_bytes()));
 274   ldr(cache, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes()));
 275   resolve_oop_handle(cache);
 276   // Add in the index
 277   // convert from field index to resolved_references() index and from
 278   // word index to byte offset. Since this is a java object, it can be compressed
 279   logical_shift_left(index, index, LogBytesPerHeapOop);
 280   add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
 281   load_heap_oop(result, Address(cache, index));
 282 }
 283 
 284 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
 285                                            Register Rcpool, Register Rindex, Register Rklass) {
 286   add(Rtemp, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord));
 287   ldrh(Rtemp, Address(Rtemp, sizeof(ConstantPool))); // Rtemp = resolved_klass_index
 288   ldr(Rklass, Address(Rcpool,  ConstantPool::resolved_klasses_offset_in_bytes())); // Rklass = cpool->_resolved_klasses
 289   add(Rklass, Rklass, AsmOperand(Rtemp, lsl, LogBytesPerWord));
 290   ldr(Rklass, Address(Rklass, Array<Klass*>::base_offset_in_bytes()));
 291 }
 292 
 293 // Generate a subtype check: branch to not_subtype if sub_klass is
 294 // not a subtype of super_klass.
 295 // Profiling code for the subtype check failure (profile_typecheck_failed)
 296 // should be explicitly generated by the caller in the not_subtype case.
 297 // Blows Rtemp, tmp1, tmp2.
 298 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
 299                                                   Register Rsuper_klass,
 300                                                   Label &not_subtype,
 301                                                   Register tmp1,
 302                                                   Register tmp2) {
 303 
 304   assert_different_registers(Rsub_klass, Rsuper_klass, tmp1, tmp2, Rtemp);
 305   Label ok_is_subtype, loop, update_cache;
 306 
 307   const Register super_check_offset = tmp1;
 308   const Register cached_super = tmp2;
 309 
 310   // Profile the not-null value's klass.
 311   profile_typecheck(tmp1, Rsub_klass);
 312 
 313   // Load the super-klass's check offset into
 314   ldr_u32(super_check_offset, Address(Rsuper_klass, Klass::super_check_offset_offset()));
 315 
 316   // Check for self
 317   cmp(Rsub_klass, Rsuper_klass);
 318 
 319   // Load from the sub-klass's super-class display list, or a 1-word cache of
 320   // the secondary superclass list, or a failing value with a sentinel offset
 321   // if the super-klass is an interface or exceptionally deep in the Java
 322   // hierarchy and we have to scan the secondary superclass list the hard way.
 323   // See if we get an immediate positive hit
 324   ldr(cached_super, Address(Rsub_klass, super_check_offset));
 325 
 326   cond_cmp(Rsuper_klass, cached_super, ne);
 327   b(ok_is_subtype, eq);
 328 
 329   // Check for immediate negative hit
 330   cmp(super_check_offset, in_bytes(Klass::secondary_super_cache_offset()));
 331   b(not_subtype, ne);
 332 
 333   // Now do a linear scan of the secondary super-klass chain.
 334   const Register supers_arr = tmp1;
 335   const Register supers_cnt = tmp2;
 336   const Register cur_super  = Rtemp;
 337 
 338   // Load objArrayOop of secondary supers.
 339   ldr(supers_arr, Address(Rsub_klass, Klass::secondary_supers_offset()));
 340 
 341   ldr_u32(supers_cnt, Address(supers_arr, Array<Klass*>::length_offset_in_bytes())); // Load the array length
 342   cmp(supers_cnt, 0);
 343 
 344   // Skip to the start of array elements and prefetch the first super-klass.
 345   ldr(cur_super, Address(supers_arr, Array<Klass*>::base_offset_in_bytes(), pre_indexed), ne);
 346   b(not_subtype, eq);
 347 
 348   bind(loop);
 349 
 350 
 351   cmp(cur_super, Rsuper_klass);
 352   b(update_cache, eq);
 353 
 354   subs(supers_cnt, supers_cnt, 1);
 355 
 356   ldr(cur_super, Address(supers_arr, wordSize, pre_indexed), ne);
 357 
 358   b(loop, ne);
 359 
 360   b(not_subtype);
 361 
 362   bind(update_cache);
 363   // Must be equal but missed in cache.  Update cache.
 364   str(Rsuper_klass, Address(Rsub_klass, Klass::secondary_super_cache_offset()));
 365 
 366   bind(ok_is_subtype);
 367 }
 368 
 369 
 370 //////////////////////////////////////////////////////////////////////////////////
 371 
 372 
 373 // Java Expression Stack
 374 
 375 void InterpreterMacroAssembler::pop_ptr(Register r) {
 376   assert(r != Rstack_top, "unpredictable instruction");
 377   ldr(r, Address(Rstack_top, wordSize, post_indexed));
 378 }
 379 
 380 void InterpreterMacroAssembler::pop_i(Register r) {
 381   assert(r != Rstack_top, "unpredictable instruction");
 382   ldr_s32(r, Address(Rstack_top, wordSize, post_indexed));
 383   zap_high_non_significant_bits(r);
 384 }
 385 
 386 void InterpreterMacroAssembler::pop_l(Register lo, Register hi) {
 387   assert_different_registers(lo, hi);
 388   assert(lo < hi, "lo must be < hi");
 389   pop(RegisterSet(lo) | RegisterSet(hi));
 390 }
 391 
 392 void InterpreterMacroAssembler::pop_f(FloatRegister fd) {
 393   fpops(fd);
 394 }
 395 
 396 void InterpreterMacroAssembler::pop_d(FloatRegister fd) {
 397   fpopd(fd);
 398 }
 399 
 400 
 401 // Transition vtos -> state. Blows R0, R1. Sets TOS cached value.
 402 void InterpreterMacroAssembler::pop(TosState state) {
 403   switch (state) {
 404     case atos: pop_ptr(R0_tos);                              break;
 405     case btos:                                               // fall through
 406     case ztos:                                               // fall through
 407     case ctos:                                               // fall through
 408     case stos:                                               // fall through
 409     case itos: pop_i(R0_tos);                                break;
 410     case ltos: pop_l(R0_tos_lo, R1_tos_hi);                  break;
 411 #ifdef __SOFTFP__
 412     case ftos: pop_i(R0_tos);                                break;
 413     case dtos: pop_l(R0_tos_lo, R1_tos_hi);                  break;
 414 #else
 415     case ftos: pop_f(S0_tos);                                break;
 416     case dtos: pop_d(D0_tos);                                break;
 417 #endif // __SOFTFP__
 418     case vtos: /* nothing to do */                           break;
 419     default  : ShouldNotReachHere();
 420   }
 421   interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 422 }
 423 
 424 void InterpreterMacroAssembler::push_ptr(Register r) {
 425   assert(r != Rstack_top, "unpredictable instruction");
 426   str(r, Address(Rstack_top, -wordSize, pre_indexed));
 427   check_stack_top_on_expansion();
 428 }
 429 
 430 void InterpreterMacroAssembler::push_i(Register r) {
 431   assert(r != Rstack_top, "unpredictable instruction");
 432   str_32(r, Address(Rstack_top, -wordSize, pre_indexed));
 433   check_stack_top_on_expansion();
 434 }
 435 
 436 void InterpreterMacroAssembler::push_l(Register lo, Register hi) {
 437   assert_different_registers(lo, hi);
 438   assert(lo < hi, "lo must be < hi");
 439   push(RegisterSet(lo) | RegisterSet(hi));
 440 }
 441 
 442 void InterpreterMacroAssembler::push_f() {
 443   fpushs(S0_tos);
 444 }
 445 
 446 void InterpreterMacroAssembler::push_d() {
 447   fpushd(D0_tos);
 448 }
 449 
 450 // Transition state -> vtos. Blows Rtemp.
 451 void InterpreterMacroAssembler::push(TosState state) {
 452   interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 453   switch (state) {
 454     case atos: push_ptr(R0_tos);                              break;
 455     case btos:                                                // fall through
 456     case ztos:                                                // fall through
 457     case ctos:                                                // fall through
 458     case stos:                                                // fall through
 459     case itos: push_i(R0_tos);                                break;
 460     case ltos: push_l(R0_tos_lo, R1_tos_hi);                  break;
 461 #ifdef __SOFTFP__
 462     case ftos: push_i(R0_tos);                                break;
 463     case dtos: push_l(R0_tos_lo, R1_tos_hi);                  break;
 464 #else
 465     case ftos: push_f();                                      break;
 466     case dtos: push_d();                                      break;
 467 #endif // __SOFTFP__
 468     case vtos: /* nothing to do */                            break;
 469     default  : ShouldNotReachHere();
 470   }
 471 }
 472 
 473 
 474 
 475 // Converts return value in R0/R1 (interpreter calling conventions) to TOS cached value.
 476 void InterpreterMacroAssembler::convert_retval_to_tos(TosState state) {
 477 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
 478   // According to interpreter calling conventions, result is returned in R0/R1,
 479   // but templates expect ftos in S0, and dtos in D0.
 480   if (state == ftos) {
 481     fmsr(S0_tos, R0);
 482   } else if (state == dtos) {
 483     fmdrr(D0_tos, R0, R1);
 484   }
 485 #endif // !__SOFTFP__ && !__ABI_HARD__
 486 }
 487 
 488 // Converts TOS cached value to return value in R0/R1 (according to interpreter calling conventions).
 489 void InterpreterMacroAssembler::convert_tos_to_retval(TosState state) {
 490 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
 491   // According to interpreter calling conventions, result is returned in R0/R1,
 492   // so ftos (S0) and dtos (D0) are moved to R0/R1.
 493   if (state == ftos) {
 494     fmrs(R0, S0_tos);
 495   } else if (state == dtos) {
 496     fmrrd(R0, R1, D0_tos);
 497   }
 498 #endif // !__SOFTFP__ && !__ABI_HARD__
 499 }
 500 
 501 
 502 
 503 // Helpers for swap and dup
 504 void InterpreterMacroAssembler::load_ptr(int n, Register val) {
 505   ldr(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n)));
 506 }
 507 
 508 void InterpreterMacroAssembler::store_ptr(int n, Register val) {
 509   str(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n)));
 510 }
 511 
 512 
 513 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() {
 514 
 515   // set sender sp
 516   mov(Rsender_sp, SP);
 517 
 518   // record last_sp
 519   str(Rsender_sp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
 520 }
 521 
 522 // Jump to from_interpreted entry of a call unless single stepping is possible
 523 // in this thread in which case we must call the i2i entry
 524 void InterpreterMacroAssembler::jump_from_interpreted(Register method) {
 525   assert_different_registers(method, Rtemp);
 526 
 527   prepare_to_jump_from_interpreted();
 528 
 529   if (can_post_interpreter_events()) {
 530     // JVMTI events, such as single-stepping, are implemented partly by avoiding running
 531     // compiled code in threads for which the event is enabled.  Check here for
 532     // interp_only_mode if these events CAN be enabled.
 533 
 534     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
 535     cmp(Rtemp, 0);
 536     ldr(PC, Address(method, Method::interpreter_entry_offset()), ne);
 537   }
 538 
 539   indirect_jump(Address(method, Method::from_interpreted_offset()), Rtemp);
 540 }
 541 
 542 
 543 void InterpreterMacroAssembler::restore_dispatch() {
 544   mov_slow(RdispatchTable, (address)Interpreter::dispatch_table(vtos));
 545 }
 546 
 547 
 548 // The following two routines provide a hook so that an implementation
 549 // can schedule the dispatch in two parts.
 550 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) {
 551   // Nothing ARM-specific to be done here.
 552 }
 553 
 554 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {
 555   dispatch_next(state, step);
 556 }
 557 
 558 void InterpreterMacroAssembler::dispatch_base(TosState state,
 559                                               DispatchTableMode table_mode,
 560                                               bool verifyoop, bool generate_poll) {
 561   if (VerifyActivationFrameSize) {
 562     Label L;
 563     sub(Rtemp, FP, SP);
 564     int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize;
 565     cmp(Rtemp, min_frame_size);
 566     b(L, ge);
 567     stop("broken stack frame");
 568     bind(L);
 569   }
 570 
 571   if (verifyoop) {
 572     interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
 573   }
 574 
 575   Label safepoint;
 576   address* const safepoint_table = Interpreter::safept_table(state);
 577   address* const table           = Interpreter::dispatch_table(state);
 578   bool needs_thread_local_poll = generate_poll && table != safepoint_table;
 579 
 580   if (needs_thread_local_poll) {
 581     NOT_PRODUCT(block_comment("Thread-local Safepoint poll"));
 582     ldr(Rtemp, Address(Rthread, JavaThread::polling_word_offset()));
 583     tbnz(Rtemp, exact_log2(SafepointMechanism::poll_bit()), safepoint);
 584   }
 585 
 586   if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) {
 587     zap_high_non_significant_bits(R0_tos);
 588   }
 589 
 590 #ifdef ASSERT
 591   Label L;
 592   mov_slow(Rtemp, (address)Interpreter::dispatch_table(vtos));
 593   cmp(Rtemp, RdispatchTable);
 594   b(L, eq);
 595   stop("invalid RdispatchTable");
 596   bind(L);
 597 #endif
 598 
 599   if (table_mode == DispatchDefault) {
 600     if (state == vtos) {
 601       indirect_jump(Address::indexed_ptr(RdispatchTable, R3_bytecode), Rtemp);
 602     } else {
 603       // on 32-bit ARM this method is faster than the one above.
 604       sub(Rtemp, RdispatchTable, (Interpreter::distance_from_dispatch_table(vtos) -
 605                            Interpreter::distance_from_dispatch_table(state)) * wordSize);
 606       indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
 607     }
 608   } else {
 609     assert(table_mode == DispatchNormal, "invalid dispatch table mode");
 610     address table = (address) Interpreter::normal_table(state);
 611     mov_slow(Rtemp, table);
 612     indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
 613   }
 614 
 615   if (needs_thread_local_poll) {
 616     bind(safepoint);
 617     lea(Rtemp, ExternalAddress((address)safepoint_table));
 618     indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
 619   }
 620 
 621   nop(); // to avoid filling CPU pipeline with invalid instructions
 622   nop();
 623 }
 624 
 625 void InterpreterMacroAssembler::dispatch_only(TosState state, bool generate_poll) {
 626   dispatch_base(state, DispatchDefault, true, generate_poll);
 627 }
 628 
 629 
 630 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) {
 631   dispatch_base(state, DispatchNormal);
 632 }
 633 
 634 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
 635   dispatch_base(state, DispatchNormal, false);
 636 }
 637 
 638 void InterpreterMacroAssembler::dispatch_next(TosState state, int step, bool generate_poll) {
 639   // load next bytecode and advance Rbcp
 640   ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed));
 641   dispatch_base(state, DispatchDefault, true, generate_poll);
 642 }
 643 
 644 void InterpreterMacroAssembler::narrow(Register result) {
 645   // mask integer result to narrower return type.
 646   const Register Rtmp = R2;
 647 
 648   // get method type
 649   ldr(Rtmp, Address(Rmethod, Method::const_offset()));
 650   ldrb(Rtmp, Address(Rtmp, ConstMethod::result_type_offset()));
 651 
 652   Label notBool, notByte, notChar, done;
 653   cmp(Rtmp, T_INT);
 654   b(done, eq);
 655 
 656   cmp(Rtmp, T_BOOLEAN);
 657   b(notBool, ne);
 658   and_32(result, result, 1);
 659   b(done);
 660 
 661   bind(notBool);
 662   cmp(Rtmp, T_BYTE);
 663   b(notByte, ne);
 664   sign_extend(result, result, 8);
 665   b(done);
 666 
 667   bind(notByte);
 668   cmp(Rtmp, T_CHAR);
 669   b(notChar, ne);
 670   zero_extend(result, result, 16);
 671   b(done);
 672 
 673   bind(notChar);
 674   // cmp(Rtmp, T_SHORT);
 675   // b(done, ne);
 676   sign_extend(result, result, 16);
 677 
 678   // Nothing to do
 679   bind(done);
 680 }
 681 
 682 // remove activation
 683 //
 684 // Unlock the receiver if this is a synchronized method.
 685 // Unlock any Java monitors from synchronized blocks.
 686 // Remove the activation from the stack.
 687 //
 688 // If there are locked Java monitors
 689 //    If throw_monitor_exception
 690 //       throws IllegalMonitorStateException
 691 //    Else if install_monitor_exception
 692 //       installs IllegalMonitorStateException
 693 //    Else
 694 //       no error processing
 695 void InterpreterMacroAssembler::remove_activation(TosState state, Register ret_addr,
 696                                                   bool throw_monitor_exception,
 697                                                   bool install_monitor_exception,
 698                                                   bool notify_jvmdi) {
 699   Label unlock, unlocked, no_unlock;
 700 
 701   // Note: Registers R0, R1, S0 and D0 (TOS cached value) may be in use for the result.
 702 
 703   const Address do_not_unlock_if_synchronized(Rthread,
 704                          JavaThread::do_not_unlock_if_synchronized_offset());
 705 
 706   const Register Rflag = R2;
 707   const Register Raccess_flags = R3;
 708 
 709   restore_method();
 710 
 711   ldrb(Rflag, do_not_unlock_if_synchronized);
 712 
 713   // get method access flags
 714   ldr_u32(Raccess_flags, Address(Rmethod, Method::access_flags_offset()));
 715 
 716   strb(zero_register(Rtemp), do_not_unlock_if_synchronized); // reset the flag
 717 
 718   // check if method is synchronized
 719 
 720   tbz(Raccess_flags, JVM_ACC_SYNCHRONIZED_BIT, unlocked);
 721 
 722   // Don't unlock anything if the _do_not_unlock_if_synchronized flag is set.
 723   cbnz(Rflag, no_unlock);
 724 
 725   // unlock monitor
 726   push(state);                                   // save result
 727 
 728   // BasicObjectLock will be first in list, since this is a synchronized method. However, need
 729   // to check that the object has not been unlocked by an explicit monitorexit bytecode.
 730 
 731   const Register Rmonitor = R0;                  // fixed in unlock_object()
 732   const Register Robj = R2;
 733 
 734   // address of first monitor
 735   sub(Rmonitor, FP, - frame::interpreter_frame_monitor_block_bottom_offset * wordSize + (int)sizeof(BasicObjectLock));
 736 
 737   ldr(Robj, Address(Rmonitor, BasicObjectLock::obj_offset_in_bytes()));
 738   cbnz(Robj, unlock);
 739 
 740   pop(state);
 741 
 742   if (throw_monitor_exception) {
 743     // Entry already unlocked, need to throw exception
 744     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
 745     should_not_reach_here();
 746   } else {
 747     // Monitor already unlocked during a stack unroll.
 748     // If requested, install an illegal_monitor_state_exception.
 749     // Continue with stack unrolling.
 750     if (install_monitor_exception) {
 751       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
 752     }
 753     b(unlocked);
 754   }
 755 
 756 
 757   // Exception case for the check that all monitors are unlocked.
 758   const Register Rcur = R2;
 759   Label restart_check_monitors_unlocked, exception_monitor_is_still_locked;
 760 
 761   bind(exception_monitor_is_still_locked);
 762   // Monitor entry is still locked, need to throw exception.
 763   // Rcur: monitor entry.
 764 
 765   if (throw_monitor_exception) {
 766     // Throw exception
 767     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
 768     should_not_reach_here();
 769   } else {
 770     // Stack unrolling. Unlock object and install illegal_monitor_exception
 771     // Unlock does not block, so don't have to worry about the frame
 772 
 773     push(state);
 774     mov(Rmonitor, Rcur);
 775     unlock_object(Rmonitor);
 776 
 777     if (install_monitor_exception) {
 778       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
 779     }
 780 
 781     pop(state);
 782     b(restart_check_monitors_unlocked);
 783   }
 784 
 785   bind(unlock);
 786   unlock_object(Rmonitor);
 787   pop(state);
 788 
 789   // Check that for block-structured locking (i.e., that all locked objects has been unlocked)
 790   bind(unlocked);
 791 
 792   // Check that all monitors are unlocked
 793   {
 794     Label loop;
 795 
 796     const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 797     const Register Rbottom = R3;
 798     const Register Rcur_obj = Rtemp;
 799 
 800     bind(restart_check_monitors_unlocked);
 801 
 802     ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
 803                                  // points to current entry, starting with top-most entry
 804     sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize);
 805                                  // points to word before bottom of monitor block
 806 
 807     cmp(Rcur, Rbottom);          // check if there are no monitors
 808     ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
 809                                  // prefetch monitor's object
 810     b(no_unlock, eq);
 811 
 812     bind(loop);
 813     // check if current entry is used
 814     cbnz(Rcur_obj, exception_monitor_is_still_locked);
 815 
 816     add(Rcur, Rcur, entry_size);      // otherwise advance to next entry
 817     cmp(Rcur, Rbottom);               // check if bottom reached
 818     ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
 819                                       // prefetch monitor's object
 820     b(loop, ne);                      // if not at bottom then check this entry
 821   }
 822 
 823   bind(no_unlock);
 824 
 825   // jvmti support
 826   if (notify_jvmdi) {
 827     notify_method_exit(state, NotifyJVMTI);     // preserve TOSCA
 828   } else {
 829     notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
 830   }
 831 
 832   // remove activation
 833   mov(Rtemp, FP);
 834   ldmia(FP, RegisterSet(FP) | RegisterSet(LR));
 835   ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize));
 836 
 837   if (ret_addr != LR) {
 838     mov(ret_addr, LR);
 839   }
 840 }
 841 
 842 
 843 // At certain points in the method invocation the monitor of
 844 // synchronized methods hasn't been entered yet.
 845 // To correctly handle exceptions at these points, we set the thread local
 846 // variable _do_not_unlock_if_synchronized to true. The remove_activation will
 847 // check this flag.
 848 void InterpreterMacroAssembler::set_do_not_unlock_if_synchronized(bool flag, Register tmp) {
 849   const Address do_not_unlock_if_synchronized(Rthread,
 850                          JavaThread::do_not_unlock_if_synchronized_offset());
 851   if (flag) {
 852     mov(tmp, 1);
 853     strb(tmp, do_not_unlock_if_synchronized);
 854   } else {
 855     strb(zero_register(tmp), do_not_unlock_if_synchronized);
 856   }
 857 }
 858 
 859 // Lock object
 860 //
 861 // Argument: R1 : Points to BasicObjectLock to be used for locking.
 862 // Must be initialized with object to lock.
 863 // Blows volatile registers R0-R3, Rtemp, LR. Calls VM.
 864 void InterpreterMacroAssembler::lock_object(Register Rlock) {
 865   assert(Rlock == R1, "the second argument");
 866 
 867   if (UseHeavyMonitors) {
 868     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
 869   } else {
 870     Label done;
 871 
 872     const Register Robj = R2;
 873     const Register Rmark = R3;
 874     assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp);
 875 
 876     const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
 877     const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
 878     const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes();
 879 
 880     Label already_locked, slow_case;
 881 
 882     // Load object pointer
 883     ldr(Robj, Address(Rlock, obj_offset));
 884 
 885     if (DiagnoseSyncOnValueBasedClasses != 0) {
 886       load_klass(R0, Robj);
 887       ldr_u32(R0, Address(R0, Klass::access_flags_offset()));
 888       tst(R0, JVM_ACC_IS_VALUE_BASED_CLASS);
 889       b(slow_case, ne);
 890     }
 891 
 892     // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
 893     // That would be acceptable as ether CAS or slow case path is taken in that case.
 894     // Exception to that is if the object is locked by the calling thread, then the recursive test will pass (guaranteed as
 895     // loads are satisfied from a store queue if performed on the same processor).
 896 
 897     assert(oopDesc::mark_offset_in_bytes() == 0, "must be");
 898     ldr(Rmark, Address(Robj, oopDesc::mark_offset_in_bytes()));
 899 
 900     // Test if object is already locked
 901     tst(Rmark, markWord::unlocked_value);
 902     b(already_locked, eq);
 903 
 904     // Save old object->mark() into BasicLock's displaced header
 905     str(Rmark, Address(Rlock, mark_offset));
 906 
 907     cas_for_lock_acquire(Rmark, Rlock, Robj, Rtemp, slow_case);
 908 
 909     b(done);
 910 
 911     // If we got here that means the object is locked by ether calling thread or another thread.
 912     bind(already_locked);
 913     // Handling of locked objects: recursive locks and slow case.
 914 
 915     // Fast check for recursive lock.
 916     //
 917     // Can apply the optimization only if this is a stack lock
 918     // allocated in this thread. For efficiency, we can focus on
 919     // recently allocated stack locks (instead of reading the stack
 920     // base and checking whether 'mark' points inside the current
 921     // thread stack):
 922     //  1) (mark & 3) == 0
 923     //  2) SP <= mark < SP + os::pagesize()
 924     //
 925     // Warning: SP + os::pagesize can overflow the stack base. We must
 926     // neither apply the optimization for an inflated lock allocated
 927     // just above the thread stack (this is why condition 1 matters)
 928     // nor apply the optimization if the stack lock is inside the stack
 929     // of another thread. The latter is avoided even in case of overflow
 930     // because we have guard pages at the end of all stacks. Hence, if
 931     // we go over the stack base and hit the stack of another thread,
 932     // this should not be in a writeable area that could contain a
 933     // stack lock allocated by that thread. As a consequence, a stack
 934     // lock less than page size away from SP is guaranteed to be
 935     // owned by the current thread.
 936     //
 937     // Note: assuming SP is aligned, we can check the low bits of
 938     // (mark-SP) instead of the low bits of mark. In that case,
 939     // assuming page size is a power of 2, we can merge the two
 940     // conditions into a single test:
 941     // => ((mark - SP) & (3 - os::pagesize())) == 0
 942 
 943     // (3 - os::pagesize()) cannot be encoded as an ARM immediate operand.
 944     // Check independently the low bits and the distance to SP.
 945     // -1- test low 2 bits
 946     movs(R0, AsmOperand(Rmark, lsl, 30));
 947     // -2- test (mark - SP) if the low two bits are 0
 948     sub(R0, Rmark, SP, eq);
 949     movs(R0, AsmOperand(R0, lsr, exact_log2(os::vm_page_size())), eq);
 950     // If still 'eq' then recursive locking OK: store 0 into lock record
 951     str(R0, Address(Rlock, mark_offset), eq);
 952 
 953     b(done, eq);
 954 
 955     bind(slow_case);
 956 
 957     // Call the runtime routine for slow case
 958     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
 959 
 960     bind(done);
 961   }
 962 }
 963 
 964 
 965 // Unlocks an object. Used in monitorexit bytecode and remove_activation.
 966 //
 967 // Argument: R0: Points to BasicObjectLock structure for lock
 968 // Throw an IllegalMonitorException if object is not locked by current thread
 969 // Blows volatile registers R0-R3, Rtemp, LR. Calls VM.
 970 void InterpreterMacroAssembler::unlock_object(Register Rlock) {
 971   assert(Rlock == R0, "the first argument");
 972 
 973   if (UseHeavyMonitors) {
 974     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock);
 975   } else {
 976     Label done, slow_case;
 977 
 978     const Register Robj = R2;
 979     const Register Rmark = R3;
 980     assert_different_registers(Robj, Rmark, Rlock, Rtemp);
 981 
 982     const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
 983     const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
 984     const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes();
 985 
 986     const Register Rzero = zero_register(Rtemp);
 987 
 988     // Load oop into Robj
 989     ldr(Robj, Address(Rlock, obj_offset));
 990 
 991     // Free entry
 992     str(Rzero, Address(Rlock, obj_offset));
 993 
 994     // Load the old header from BasicLock structure
 995     ldr(Rmark, Address(Rlock, mark_offset));
 996 
 997     // Test for recursion (zero mark in BasicLock)
 998     cbz(Rmark, done);
 999 
1000     bool allow_fallthrough_on_failure = true;
1001 
1002     cas_for_lock_release(Rlock, Rmark, Robj, Rtemp, slow_case, allow_fallthrough_on_failure);
1003 
1004     b(done, eq);
1005 
1006     bind(slow_case);
1007 
1008     // Call the runtime routine for slow case.
1009     str(Robj, Address(Rlock, obj_offset)); // restore obj
1010     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock);
1011 
1012     bind(done);
1013   }
1014 }
1015 
1016 
1017 // Test ImethodDataPtr.  If it is null, continue at the specified label
1018 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) {
1019   assert(ProfileInterpreter, "must be profiling interpreter");
1020   ldr(mdp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1021   cbz(mdp, zero_continue);
1022 }
1023 
1024 
1025 // Set the method data pointer for the current bcp.
1026 // Blows volatile registers R0-R3, Rtemp, LR.
1027 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
1028   assert(ProfileInterpreter, "must be profiling interpreter");
1029   Label set_mdp;
1030 
1031   // Test MDO to avoid the call if it is NULL.
1032   ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
1033   cbz(Rtemp, set_mdp);
1034 
1035   mov(R0, Rmethod);
1036   mov(R1, Rbcp);
1037   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), R0, R1);
1038   // R0/W0: mdi
1039 
1040   // mdo is guaranteed to be non-zero here, we checked for it before the call.
1041   ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
1042   add(Rtemp, Rtemp, in_bytes(MethodData::data_offset()));
1043   add_ptr_scaled_int32(Rtemp, Rtemp, R0, 0);
1044 
1045   bind(set_mdp);
1046   str(Rtemp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1047 }
1048 
1049 
1050 void InterpreterMacroAssembler::verify_method_data_pointer() {
1051   assert(ProfileInterpreter, "must be profiling interpreter");
1052 #ifdef ASSERT
1053   Label verify_continue;
1054   save_caller_save_registers();
1055 
1056   const Register Rmdp = R2;
1057   test_method_data_pointer(Rmdp, verify_continue); // If mdp is zero, continue
1058 
1059   // If the mdp is valid, it will point to a DataLayout header which is
1060   // consistent with the bcp.  The converse is highly probable also.
1061 
1062   ldrh(R3, Address(Rmdp, DataLayout::bci_offset()));
1063   ldr(Rtemp, Address(Rmethod, Method::const_offset()));
1064   add(R3, R3, Rtemp);
1065   add(R3, R3, in_bytes(ConstMethod::codes_offset()));
1066   cmp(R3, Rbcp);
1067   b(verify_continue, eq);
1068 
1069   mov(R0, Rmethod);
1070   mov(R1, Rbcp);
1071   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R0, R1, Rmdp);
1072 
1073   bind(verify_continue);
1074   restore_caller_save_registers();
1075 #endif // ASSERT
1076 }
1077 
1078 
1079 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, int offset, Register value) {
1080   assert(ProfileInterpreter, "must be profiling interpreter");
1081   assert_different_registers(mdp_in, value);
1082   str(value, Address(mdp_in, offset));
1083 }
1084 
1085 
1086 // Increments mdp data. Sets bumped_count register to adjusted counter.
1087 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
1088                                                       int offset,
1089                                                       Register bumped_count,
1090                                                       bool decrement) {
1091   assert(ProfileInterpreter, "must be profiling interpreter");
1092 
1093   // Counter address
1094   Address data(mdp_in, offset);
1095   assert_different_registers(mdp_in, bumped_count);
1096 
1097   increment_mdp_data_at(data, bumped_count, decrement);
1098 }
1099 
1100 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, int flag_byte_constant) {
1101   assert_different_registers(mdp_in, Rtemp);
1102   assert(ProfileInterpreter, "must be profiling interpreter");
1103   assert((0 < flag_byte_constant) && (flag_byte_constant < (1 << BitsPerByte)), "flag mask is out of range");
1104 
1105   // Set the flag
1106   ldrb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset())));
1107   orr(Rtemp, Rtemp, (unsigned)flag_byte_constant);
1108   strb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset())));
1109 }
1110 
1111 
1112 // Increments mdp data. Sets bumped_count register to adjusted counter.
1113 void InterpreterMacroAssembler::increment_mdp_data_at(Address data,
1114                                                       Register bumped_count,
1115                                                       bool decrement) {
1116   assert(ProfileInterpreter, "must be profiling interpreter");
1117 
1118   ldr(bumped_count, data);
1119   if (decrement) {
1120     // Decrement the register. Set condition codes.
1121     subs(bumped_count, bumped_count, DataLayout::counter_increment);
1122     // Avoid overflow.
1123     add(bumped_count, bumped_count, DataLayout::counter_increment, pl);
1124   } else {
1125     // Increment the register. Set condition codes.
1126     adds(bumped_count, bumped_count, DataLayout::counter_increment);
1127     // Avoid overflow.
1128     sub(bumped_count, bumped_count, DataLayout::counter_increment, mi);
1129   }
1130   str(bumped_count, data);
1131 }
1132 
1133 
1134 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in,
1135                                                  int offset,
1136                                                  Register value,
1137                                                  Register test_value_out,
1138                                                  Label& not_equal_continue) {
1139   assert(ProfileInterpreter, "must be profiling interpreter");
1140   assert_different_registers(mdp_in, test_value_out, value);
1141 
1142   ldr(test_value_out, Address(mdp_in, offset));
1143   cmp(test_value_out, value);
1144 
1145   b(not_equal_continue, ne);
1146 }
1147 
1148 
1149 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, int offset_of_disp, Register reg_temp) {
1150   assert(ProfileInterpreter, "must be profiling interpreter");
1151   assert_different_registers(mdp_in, reg_temp);
1152 
1153   ldr(reg_temp, Address(mdp_in, offset_of_disp));
1154   add(mdp_in, mdp_in, reg_temp);
1155   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1156 }
1157 
1158 
1159 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, Register reg_offset, Register reg_tmp) {
1160   assert(ProfileInterpreter, "must be profiling interpreter");
1161   assert_different_registers(mdp_in, reg_offset, reg_tmp);
1162 
1163   ldr(reg_tmp, Address(mdp_in, reg_offset));
1164   add(mdp_in, mdp_in, reg_tmp);
1165   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1166 }
1167 
1168 
1169 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, int constant) {
1170   assert(ProfileInterpreter, "must be profiling interpreter");
1171   add(mdp_in, mdp_in, constant);
1172   str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1173 }
1174 
1175 
1176 // Blows volatile registers R0-R3, Rtemp, LR).
1177 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {
1178   assert(ProfileInterpreter, "must be profiling interpreter");
1179   assert_different_registers(return_bci, R0, R1, R2, R3, Rtemp);
1180 
1181   mov(R1, return_bci);
1182   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), R1);
1183 }
1184 
1185 
1186 // Sets mdp, bumped_count registers, blows Rtemp.
1187 void InterpreterMacroAssembler::profile_taken_branch(Register mdp, Register bumped_count) {
1188   assert_different_registers(mdp, bumped_count);
1189 
1190   if (ProfileInterpreter) {
1191     Label profile_continue;
1192 
1193     // If no method data exists, go to profile_continue.
1194     // Otherwise, assign to mdp
1195     test_method_data_pointer(mdp, profile_continue);
1196 
1197     // We are taking a branch. Increment the taken count.
1198     increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()), bumped_count);
1199 
1200     // The method data pointer needs to be updated to reflect the new target.
1201     update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()), Rtemp);
1202 
1203     bind (profile_continue);
1204   }
1205 }
1206 
1207 
1208 // Sets mdp, blows Rtemp.
1209 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
1210   assert_different_registers(mdp, Rtemp);
1211 
1212   if (ProfileInterpreter) {
1213     Label profile_continue;
1214 
1215     // If no method data exists, go to profile_continue.
1216     test_method_data_pointer(mdp, profile_continue);
1217 
1218     // We are taking a branch.  Increment the not taken count.
1219     increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()), Rtemp);
1220 
1221     // The method data pointer needs to be updated to correspond to the next bytecode
1222     update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
1223 
1224     bind (profile_continue);
1225   }
1226 }
1227 
1228 
1229 // Sets mdp, blows Rtemp.
1230 void InterpreterMacroAssembler::profile_call(Register mdp) {
1231   assert_different_registers(mdp, Rtemp);
1232 
1233   if (ProfileInterpreter) {
1234     Label profile_continue;
1235 
1236     // If no method data exists, go to profile_continue.
1237     test_method_data_pointer(mdp, profile_continue);
1238 
1239     // We are making a call.  Increment the count.
1240     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1241 
1242     // The method data pointer needs to be updated to reflect the new target.
1243     update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1244 
1245     bind (profile_continue);
1246   }
1247 }
1248 
1249 
1250 // Sets mdp, blows Rtemp.
1251 void InterpreterMacroAssembler::profile_final_call(Register mdp) {
1252   if (ProfileInterpreter) {
1253     Label profile_continue;
1254 
1255     // If no method data exists, go to profile_continue.
1256     test_method_data_pointer(mdp, profile_continue);
1257 
1258     // We are making a call.  Increment the count.
1259     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1260 
1261     // The method data pointer needs to be updated to reflect the new target.
1262     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1263 
1264     bind (profile_continue);
1265   }
1266 }
1267 
1268 
1269 // Sets mdp, blows Rtemp.
1270 void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) {
1271   assert_different_registers(mdp, receiver, Rtemp);
1272 
1273   if (ProfileInterpreter) {
1274     Label profile_continue;
1275 
1276     // If no method data exists, go to profile_continue.
1277     test_method_data_pointer(mdp, profile_continue);
1278 
1279     Label skip_receiver_profile;
1280     if (receiver_can_be_null) {
1281       Label not_null;
1282       cbnz(receiver, not_null);
1283       // We are making a call.  Increment the count for null receiver.
1284       increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1285       b(skip_receiver_profile);
1286       bind(not_null);
1287     }
1288 
1289     // Record the receiver type.
1290     record_klass_in_profile(receiver, mdp, Rtemp, true);
1291     bind(skip_receiver_profile);
1292 
1293     // The method data pointer needs to be updated to reflect the new target.
1294     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1295     bind(profile_continue);
1296   }
1297 }
1298 
1299 
1300 void InterpreterMacroAssembler::record_klass_in_profile_helper(
1301                                         Register receiver, Register mdp,
1302                                         Register reg_tmp,
1303                                         int start_row, Label& done, bool is_virtual_call) {
1304   if (TypeProfileWidth == 0)
1305     return;
1306 
1307   assert_different_registers(receiver, mdp, reg_tmp);
1308 
1309   int last_row = VirtualCallData::row_limit() - 1;
1310   assert(start_row <= last_row, "must be work left to do");
1311   // Test this row for both the receiver and for null.
1312   // Take any of three different outcomes:
1313   //   1. found receiver => increment count and goto done
1314   //   2. found null => keep looking for case 1, maybe allocate this cell
1315   //   3. found something else => keep looking for cases 1 and 2
1316   // Case 3 is handled by a recursive call.
1317   for (int row = start_row; row <= last_row; row++) {
1318     Label next_test;
1319 
1320     // See if the receiver is receiver[n].
1321     int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row));
1322 
1323     test_mdp_data_at(mdp, recvr_offset, receiver, reg_tmp, next_test);
1324 
1325     // The receiver is receiver[n].  Increment count[n].
1326     int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row));
1327     increment_mdp_data_at(mdp, count_offset, reg_tmp);
1328     b(done);
1329 
1330     bind(next_test);
1331     // reg_tmp now contains the receiver from the CallData.
1332 
1333     if (row == start_row) {
1334       Label found_null;
1335       // Failed the equality check on receiver[n]...  Test for null.
1336       if (start_row == last_row) {
1337         // The only thing left to do is handle the null case.
1338         if (is_virtual_call) {
1339           cbz(reg_tmp, found_null);
1340           // Receiver did not match any saved receiver and there is no empty row for it.
1341           // Increment total counter to indicate polymorphic case.
1342           increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), reg_tmp);
1343           b(done);
1344           bind(found_null);
1345         } else {
1346           cbnz(reg_tmp, done);
1347         }
1348         break;
1349       }
1350       // Since null is rare, make it be the branch-taken case.
1351       cbz(reg_tmp, found_null);
1352 
1353       // Put all the "Case 3" tests here.
1354       record_klass_in_profile_helper(receiver, mdp, reg_tmp, start_row + 1, done, is_virtual_call);
1355 
1356       // Found a null.  Keep searching for a matching receiver,
1357       // but remember that this is an empty (unused) slot.
1358       bind(found_null);
1359     }
1360   }
1361 
1362   // In the fall-through case, we found no matching receiver, but we
1363   // observed the receiver[start_row] is NULL.
1364 
1365   // Fill in the receiver field and increment the count.
1366   int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row));
1367   set_mdp_data_at(mdp, recvr_offset, receiver);
1368   int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row));
1369   mov(reg_tmp, DataLayout::counter_increment);
1370   set_mdp_data_at(mdp, count_offset, reg_tmp);
1371   if (start_row > 0) {
1372     b(done);
1373   }
1374 }
1375 
1376 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
1377                                                         Register mdp,
1378                                                         Register reg_tmp,
1379                                                         bool is_virtual_call) {
1380   assert(ProfileInterpreter, "must be profiling");
1381   assert_different_registers(receiver, mdp, reg_tmp);
1382 
1383   Label done;
1384 
1385   record_klass_in_profile_helper(receiver, mdp, reg_tmp, 0, done, is_virtual_call);
1386 
1387   bind (done);
1388 }
1389 
1390 // Sets mdp, blows volatile registers R0-R3, Rtemp, LR).
1391 void InterpreterMacroAssembler::profile_ret(Register mdp, Register return_bci) {
1392   assert_different_registers(mdp, return_bci, Rtemp, R0, R1, R2, R3);
1393 
1394   if (ProfileInterpreter) {
1395     Label profile_continue;
1396     uint row;
1397 
1398     // If no method data exists, go to profile_continue.
1399     test_method_data_pointer(mdp, profile_continue);
1400 
1401     // Update the total ret count.
1402     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1403 
1404     for (row = 0; row < RetData::row_limit(); row++) {
1405       Label next_test;
1406 
1407       // See if return_bci is equal to bci[n]:
1408       test_mdp_data_at(mdp, in_bytes(RetData::bci_offset(row)), return_bci,
1409                        Rtemp, next_test);
1410 
1411       // return_bci is equal to bci[n].  Increment the count.
1412       increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)), Rtemp);
1413 
1414       // The method data pointer needs to be updated to reflect the new target.
1415       update_mdp_by_offset(mdp, in_bytes(RetData::bci_displacement_offset(row)), Rtemp);
1416       b(profile_continue);
1417       bind(next_test);
1418     }
1419 
1420     update_mdp_for_ret(return_bci);
1421 
1422     bind(profile_continue);
1423   }
1424 }
1425 
1426 
1427 // Sets mdp.
1428 void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
1429   if (ProfileInterpreter) {
1430     Label profile_continue;
1431 
1432     // If no method data exists, go to profile_continue.
1433     test_method_data_pointer(mdp, profile_continue);
1434 
1435     set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1436 
1437     // The method data pointer needs to be updated.
1438     int mdp_delta = in_bytes(BitData::bit_data_size());
1439     if (TypeProfileCasts) {
1440       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1441     }
1442     update_mdp_by_constant(mdp, mdp_delta);
1443 
1444     bind (profile_continue);
1445   }
1446 }
1447 
1448 
1449 // Sets mdp, blows Rtemp.
1450 void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) {
1451   assert_different_registers(mdp, Rtemp);
1452 
1453   if (ProfileInterpreter && TypeProfileCasts) {
1454     Label profile_continue;
1455 
1456     // If no method data exists, go to profile_continue.
1457     test_method_data_pointer(mdp, profile_continue);
1458 
1459     int count_offset = in_bytes(CounterData::count_offset());
1460     // Back up the address, since we have already bumped the mdp.
1461     count_offset -= in_bytes(VirtualCallData::virtual_call_data_size());
1462 
1463     // *Decrement* the counter.  We expect to see zero or small negatives.
1464     increment_mdp_data_at(mdp, count_offset, Rtemp, true);
1465 
1466     bind (profile_continue);
1467   }
1468 }
1469 
1470 
1471 // Sets mdp, blows Rtemp.
1472 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass)
1473 {
1474   assert_different_registers(mdp, klass, Rtemp);
1475 
1476   if (ProfileInterpreter) {
1477     Label profile_continue;
1478 
1479     // If no method data exists, go to profile_continue.
1480     test_method_data_pointer(mdp, profile_continue);
1481 
1482     // The method data pointer needs to be updated.
1483     int mdp_delta = in_bytes(BitData::bit_data_size());
1484     if (TypeProfileCasts) {
1485       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1486 
1487       // Record the object type.
1488       record_klass_in_profile(klass, mdp, Rtemp, false);
1489     }
1490     update_mdp_by_constant(mdp, mdp_delta);
1491 
1492     bind(profile_continue);
1493   }
1494 }
1495 
1496 
1497 // Sets mdp, blows Rtemp.
1498 void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
1499   assert_different_registers(mdp, Rtemp);
1500 
1501   if (ProfileInterpreter) {
1502     Label profile_continue;
1503 
1504     // If no method data exists, go to profile_continue.
1505     test_method_data_pointer(mdp, profile_continue);
1506 
1507     // Update the default case count
1508     increment_mdp_data_at(mdp, in_bytes(MultiBranchData::default_count_offset()), Rtemp);
1509 
1510     // The method data pointer needs to be updated.
1511     update_mdp_by_offset(mdp, in_bytes(MultiBranchData::default_displacement_offset()), Rtemp);
1512 
1513     bind(profile_continue);
1514   }
1515 }
1516 
1517 
1518 // Sets mdp. Blows reg_tmp1, reg_tmp2. Index could be the same as reg_tmp2.
1519 void InterpreterMacroAssembler::profile_switch_case(Register mdp, Register index, Register reg_tmp1, Register reg_tmp2) {
1520   assert_different_registers(mdp, reg_tmp1, reg_tmp2);
1521   assert_different_registers(mdp, reg_tmp1, index);
1522 
1523   if (ProfileInterpreter) {
1524     Label profile_continue;
1525 
1526     const int count_offset = in_bytes(MultiBranchData::case_array_offset()) +
1527                               in_bytes(MultiBranchData::relative_count_offset());
1528 
1529     const int displacement_offset = in_bytes(MultiBranchData::case_array_offset()) +
1530                               in_bytes(MultiBranchData::relative_displacement_offset());
1531 
1532     // If no method data exists, go to profile_continue.
1533     test_method_data_pointer(mdp, profile_continue);
1534 
1535     // Build the base (index * per_case_size_in_bytes())
1536     logical_shift_left(reg_tmp1, index, exact_log2(in_bytes(MultiBranchData::per_case_size())));
1537 
1538     // Update the case count
1539     add(reg_tmp1, reg_tmp1, count_offset);
1540     increment_mdp_data_at(Address(mdp, reg_tmp1), reg_tmp2);
1541 
1542     // The method data pointer needs to be updated.
1543     add(reg_tmp1, reg_tmp1, displacement_offset - count_offset);
1544     update_mdp_by_offset(mdp, reg_tmp1, reg_tmp2);
1545 
1546     bind (profile_continue);
1547   }
1548 }
1549 
1550 
1551 void InterpreterMacroAssembler::byteswap_u32(Register r, Register rtmp1, Register rtmp2) {
1552   if (VM_Version::supports_rev()) {
1553     rev(r, r);
1554   } else {
1555     eor(rtmp1, r, AsmOperand(r, ror, 16));
1556     mvn(rtmp2, 0x0000ff00);
1557     andr(rtmp1, rtmp2, AsmOperand(rtmp1, lsr, 8));
1558     eor(r, rtmp1, AsmOperand(r, ror, 8));
1559   }
1560 }
1561 
1562 
1563 void InterpreterMacroAssembler::inc_global_counter(address address_of_counter, int offset, Register tmp1, Register tmp2, bool avoid_overflow) {
1564   const intx addr = (intx) (address_of_counter + offset);
1565 
1566   assert ((addr & 0x3) == 0, "address of counter should be aligned");
1567   const intx offset_mask = right_n_bits(12);
1568 
1569   const address base = (address) (addr & ~offset_mask);
1570   const int offs = (int) (addr & offset_mask);
1571 
1572   const Register addr_base = tmp1;
1573   const Register val = tmp2;
1574 
1575   mov_slow(addr_base, base);
1576   ldr_s32(val, Address(addr_base, offs));
1577 
1578   if (avoid_overflow) {
1579     adds_32(val, val, 1);
1580     str(val, Address(addr_base, offs), pl);
1581   } else {
1582     add_32(val, val, 1);
1583     str_32(val, Address(addr_base, offs));
1584   }
1585 }
1586 
1587 void InterpreterMacroAssembler::interp_verify_oop(Register reg, TosState state, const char *file, int line) {
1588   if (state == atos) { MacroAssembler::_verify_oop(reg, "broken oop", file, line); }
1589 }
1590 
1591 // Inline assembly for:
1592 //
1593 // if (thread is in interp_only_mode) {
1594 //   InterpreterRuntime::post_method_entry();
1595 // }
1596 // if (DTraceMethodProbes) {
1597 //   SharedRuntime::dtrace_method_entry(method, receiver);
1598 // }
1599 // if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) {
1600 //   SharedRuntime::rc_trace_method_entry(method, receiver);
1601 // }
1602 
1603 void InterpreterMacroAssembler::notify_method_entry() {
1604   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1605   // track stack depth.  If it is possible to enter interp_only_mode we add
1606   // the code to check if the event should be sent.
1607   if (can_post_interpreter_events()) {
1608     Label L;
1609 
1610     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
1611     cbz(Rtemp, L);
1612 
1613     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry));
1614 
1615     bind(L);
1616   }
1617 
1618   // Note: Disable DTrace runtime check for now to eliminate overhead on each method entry
1619   if (DTraceMethodProbes) {
1620     Label Lcontinue;
1621 
1622     ldrb_global(Rtemp, (address)&DTraceMethodProbes);
1623     cbz(Rtemp, Lcontinue);
1624 
1625     mov(R0, Rthread);
1626     mov(R1, Rmethod);
1627     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), R0, R1);
1628 
1629     bind(Lcontinue);
1630   }
1631   // RedefineClasses() tracing support for obsolete method entry
1632   if (log_is_enabled(Trace, redefine, class, obsolete)) {
1633     mov(R0, Rthread);
1634     mov(R1, Rmethod);
1635     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
1636                  R0, R1);
1637   }
1638 }
1639 
1640 
1641 void InterpreterMacroAssembler::notify_method_exit(
1642                  TosState state, NotifyMethodExitMode mode,
1643                  bool native, Register result_lo, Register result_hi, FloatRegister result_fp) {
1644   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1645   // track stack depth.  If it is possible to enter interp_only_mode we add
1646   // the code to check if the event should be sent.
1647   if (mode == NotifyJVMTI && can_post_interpreter_events()) {
1648     Label L;
1649     // Note: frame::interpreter_frame_result has a dependency on how the
1650     // method result is saved across the call to post_method_exit. If this
1651     // is changed then the interpreter_frame_result implementation will
1652     // need to be updated too.
1653 
1654     ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
1655     cbz(Rtemp, L);
1656 
1657     if (native) {
1658       // For c++ and template interpreter push both result registers on the
1659       // stack in native, we don't know the state.
1660       // See frame::interpreter_frame_result for code that gets the result values from here.
1661       assert(result_lo != noreg, "result registers should be defined");
1662 
1663       assert(result_hi != noreg, "result registers should be defined");
1664 
1665 #ifdef __ABI_HARD__
1666       assert(result_fp != fnoreg, "FP result register must be defined");
1667       sub(SP, SP, 2 * wordSize);
1668       fstd(result_fp, Address(SP));
1669 #endif // __ABI_HARD__
1670 
1671       push(RegisterSet(result_lo) | RegisterSet(result_hi));
1672 
1673       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
1674 
1675       pop(RegisterSet(result_lo) | RegisterSet(result_hi));
1676 #ifdef __ABI_HARD__
1677       fldd(result_fp, Address(SP));
1678       add(SP, SP, 2 * wordSize);
1679 #endif // __ABI_HARD__
1680 
1681     } else {
1682       // For the template interpreter, the value on tos is the size of the
1683       // state. (c++ interpreter calls jvmti somewhere else).
1684       push(state);
1685       call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
1686       pop(state);
1687     }
1688 
1689     bind(L);
1690   }
1691 
1692   // Note: Disable DTrace runtime check for now to eliminate overhead on each method exit
1693   if (DTraceMethodProbes) {
1694     Label Lcontinue;
1695 
1696     ldrb_global(Rtemp, (address)&DTraceMethodProbes);
1697     cbz(Rtemp, Lcontinue);
1698 
1699     push(state);
1700 
1701     mov(R0, Rthread);
1702     mov(R1, Rmethod);
1703 
1704     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), R0, R1);
1705 
1706     pop(state);
1707 
1708     bind(Lcontinue);
1709   }
1710 }
1711 
1712 
1713 #ifndef PRODUCT
1714 
1715 void InterpreterMacroAssembler::trace_state(const char* msg) {
1716   int push_size = save_caller_save_registers();
1717 
1718   Label Lcontinue;
1719   InlinedString Lmsg0("%s: FP=" INTPTR_FORMAT ", SP=" INTPTR_FORMAT "\n");
1720   InlinedString Lmsg(msg);
1721   InlinedAddress Lprintf((address)printf);
1722 
1723   ldr_literal(R0, Lmsg0);
1724   ldr_literal(R1, Lmsg);
1725   mov(R2, FP);
1726   add(R3, SP, push_size);  // original SP (without saved registers)
1727   ldr_literal(Rtemp, Lprintf);
1728   call(Rtemp);
1729 
1730   b(Lcontinue);
1731 
1732   bind_literal(Lmsg0);
1733   bind_literal(Lmsg);
1734   bind_literal(Lprintf);
1735 
1736 
1737   bind(Lcontinue);
1738 
1739   restore_caller_save_registers();
1740 }
1741 
1742 #endif
1743 
1744 // Jump if ((*counter_addr += increment) & mask) satisfies the condition.
1745 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
1746                                                         int increment, Address mask_addr,
1747                                                         Register scratch, Register scratch2,
1748                                                         AsmCondition cond, Label* where) {
1749   // caution: scratch2 and base address of counter_addr can be the same
1750   assert_different_registers(scratch, scratch2);
1751   ldr_u32(scratch, counter_addr);
1752   add(scratch, scratch, increment);
1753   str_32(scratch, counter_addr);
1754 
1755   ldr(scratch2, mask_addr);
1756   andrs(scratch, scratch, scratch2);
1757   b(*where, cond);
1758 }
1759 
1760 void InterpreterMacroAssembler::get_method_counters(Register method,
1761                                                     Register Rcounters,
1762                                                     Label& skip,
1763                                                     bool saveRegs,
1764                                                     Register reg1,
1765                                                     Register reg2,
1766                                                     Register reg3) {
1767   const Address method_counters(method, Method::method_counters_offset());
1768   Label has_counters;
1769 
1770   ldr(Rcounters, method_counters);
1771   cbnz(Rcounters, has_counters);
1772 
1773   if (saveRegs) {
1774     // Save and restore in use caller-saved registers since they will be trashed by call_VM
1775     assert(reg1 != noreg, "must specify reg1");
1776     assert(reg2 != noreg, "must specify reg2");
1777     assert(reg3 == noreg, "must not specify reg3");
1778     push(RegisterSet(reg1) | RegisterSet(reg2));
1779   }
1780 
1781   mov(R1, method);
1782   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::build_method_counters), R1);
1783 
1784   if (saveRegs) {
1785     pop(RegisterSet(reg1) | RegisterSet(reg2));
1786   }
1787 
1788   ldr(Rcounters, method_counters);
1789   cbz(Rcounters, skip); // No MethodCounters created, OutOfMemory
1790 
1791   bind(has_counters);
1792 }