1 /*
   2  * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "compiler/compiler_globals.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/barrierSetAssembler.hpp"
  30 #include "interp_masm_aarch64.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "interpreter/interpreterRuntime.hpp"
  33 #include "logging/log.hpp"
  34 #include "oops/arrayOop.hpp"
  35 #include "oops/constMethodFlags.hpp"
  36 #include "oops/markWord.hpp"
  37 #include "oops/method.hpp"
  38 #include "oops/methodData.hpp"
  39 #include "oops/inlineKlass.hpp"
  40 #include "oops/resolvedFieldEntry.hpp"
  41 #include "oops/resolvedIndyEntry.hpp"
  42 #include "oops/resolvedMethodEntry.hpp"
  43 #include "prims/jvmtiExport.hpp"
  44 #include "prims/jvmtiThreadState.hpp"
  45 #include "runtime/basicLock.hpp"
  46 #include "runtime/frame.inline.hpp"
  47 #include "runtime/javaThread.hpp"
  48 #include "runtime/safepointMechanism.hpp"
  49 #include "runtime/sharedRuntime.hpp"
  50 #include "utilities/powerOfTwo.hpp"
  51 
  52 void InterpreterMacroAssembler::narrow(Register result) {
  53 
  54   // Get method->_constMethod->_result_type
  55   ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
  56   ldr(rscratch1, Address(rscratch1, Method::const_offset()));
  57   ldrb(rscratch1, Address(rscratch1, ConstMethod::result_type_offset()));
  58 
  59   Label done, notBool, notByte, notChar;
  60 
  61   // common case first
  62   cmpw(rscratch1, T_INT);
  63   br(Assembler::EQ, done);
  64 
  65   // mask integer result to narrower return type.
  66   cmpw(rscratch1, T_BOOLEAN);
  67   br(Assembler::NE, notBool);
  68   andw(result, result, 0x1);
  69   b(done);
  70 
  71   bind(notBool);
  72   cmpw(rscratch1, T_BYTE);
  73   br(Assembler::NE, notByte);
  74   sbfx(result, result, 0, 8);
  75   b(done);
  76 
  77   bind(notByte);
  78   cmpw(rscratch1, T_CHAR);
  79   br(Assembler::NE, notChar);
  80   ubfx(result, result, 0, 16);  // truncate upper 16 bits
  81   b(done);
  82 
  83   bind(notChar);
  84   sbfx(result, result, 0, 16);     // sign-extend short
  85 
  86   // Nothing to do for T_INT
  87   bind(done);
  88 }
  89 
  90 void InterpreterMacroAssembler::jump_to_entry(address entry) {
  91   assert(entry, "Entry must have been generated by now");
  92   b(entry);
  93 }
  94 
  95 void InterpreterMacroAssembler::check_and_handle_popframe(Register java_thread) {
  96   if (JvmtiExport::can_pop_frame()) {
  97     Label L;
  98     // Initiate popframe handling only if it is not already being
  99     // processed.  If the flag has the popframe_processing bit set, it
 100     // means that this code is called *during* popframe handling - we
 101     // don't want to reenter.
 102     // This method is only called just after the call into the vm in
 103     // call_VM_base, so the arg registers are available.
 104     ldrw(rscratch1, Address(rthread, JavaThread::popframe_condition_offset()));
 105     tbz(rscratch1, exact_log2(JavaThread::popframe_pending_bit), L);
 106     tbnz(rscratch1, exact_log2(JavaThread::popframe_processing_bit), L);
 107     // Call Interpreter::remove_activation_preserving_args_entry() to get the
 108     // address of the same-named entrypoint in the generated interpreter code.
 109     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));
 110     br(r0);
 111     bind(L);
 112   }
 113 }
 114 
 115 
 116 void InterpreterMacroAssembler::load_earlyret_value(TosState state) {
 117   ldr(r2, Address(rthread, JavaThread::jvmti_thread_state_offset()));
 118   const Address tos_addr(r2, JvmtiThreadState::earlyret_tos_offset());
 119   const Address oop_addr(r2, JvmtiThreadState::earlyret_oop_offset());
 120   const Address val_addr(r2, JvmtiThreadState::earlyret_value_offset());
 121   switch (state) {
 122     case atos: ldr(r0, oop_addr);
 123                str(zr, oop_addr);
 124                interp_verify_oop(r0, state);        break;
 125     case ltos: ldr(r0, val_addr);                   break;
 126     case btos:                                   // fall through
 127     case ztos:                                   // fall through
 128     case ctos:                                   // fall through
 129     case stos:                                   // fall through
 130     case itos: ldrw(r0, val_addr);                  break;
 131     case ftos: ldrs(v0, val_addr);                  break;
 132     case dtos: ldrd(v0, val_addr);                  break;
 133     case vtos: /* nothing to do */                  break;
 134     default  : ShouldNotReachHere();
 135   }
 136   // Clean up tos value in the thread object
 137   movw(rscratch1, (int) ilgl);
 138   strw(rscratch1, tos_addr);
 139   strw(zr, val_addr);
 140 }
 141 
 142 
 143 void InterpreterMacroAssembler::check_and_handle_earlyret(Register java_thread) {
 144   if (JvmtiExport::can_force_early_return()) {
 145     Label L;
 146     ldr(rscratch1, Address(rthread, JavaThread::jvmti_thread_state_offset()));
 147     cbz(rscratch1, L); // if (thread->jvmti_thread_state() == nullptr) exit;
 148 
 149     // Initiate earlyret handling only if it is not already being processed.
 150     // If the flag has the earlyret_processing bit set, it means that this code
 151     // is called *during* earlyret handling - we don't want to reenter.
 152     ldrw(rscratch1, Address(rscratch1, JvmtiThreadState::earlyret_state_offset()));
 153     cmpw(rscratch1, JvmtiThreadState::earlyret_pending);
 154     br(Assembler::NE, L);
 155 
 156     // Call Interpreter::remove_activation_early_entry() to get the address of the
 157     // same-named entrypoint in the generated interpreter code.
 158     ldr(rscratch1, Address(rthread, JavaThread::jvmti_thread_state_offset()));
 159     ldrw(rscratch1, Address(rscratch1, JvmtiThreadState::earlyret_tos_offset()));
 160     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), rscratch1);
 161     br(r0);
 162     bind(L);
 163   }
 164 }
 165 
 166 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(
 167   Register reg,
 168   int bcp_offset) {
 169   assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");
 170   ldrh(reg, Address(rbcp, bcp_offset));
 171   rev16(reg, reg);
 172 }
 173 
 174 void InterpreterMacroAssembler::get_dispatch() {
 175   uint64_t offset;
 176   adrp(rdispatch, ExternalAddress((address)Interpreter::dispatch_table()), offset);
 177   // Use add() here after ARDP, rather than lea().
 178   // lea() does not generate anything if its offset is zero.
 179   // However, relocs expect to find either an ADD or a load/store
 180   // insn after an ADRP.  add() always generates an ADD insn, even
 181   // for add(Rn, Rn, 0).
 182   add(rdispatch, rdispatch, offset);
 183 }
 184 
 185 void InterpreterMacroAssembler::get_cache_index_at_bcp(Register index,
 186                                                        int bcp_offset,
 187                                                        size_t index_size) {
 188   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 189   if (index_size == sizeof(u2)) {
 190     load_unsigned_short(index, Address(rbcp, bcp_offset));
 191   } else if (index_size == sizeof(u4)) {
 192     // assert(EnableInvokeDynamic, "giant index used only for JSR 292");
 193     ldrw(index, Address(rbcp, bcp_offset));
 194   } else if (index_size == sizeof(u1)) {
 195     load_unsigned_byte(index, Address(rbcp, bcp_offset));
 196   } else {
 197     ShouldNotReachHere();
 198   }
 199 }
 200 
 201 void InterpreterMacroAssembler::get_method_counters(Register method,
 202                                                     Register mcs, Label& skip) {
 203   Label has_counters;
 204   ldr(mcs, Address(method, Method::method_counters_offset()));
 205   cbnz(mcs, has_counters);
 206   call_VM(noreg, CAST_FROM_FN_PTR(address,
 207           InterpreterRuntime::build_method_counters), method);
 208   ldr(mcs, Address(method, Method::method_counters_offset()));
 209   cbz(mcs, skip); // No MethodCounters allocated, OutOfMemory
 210   bind(has_counters);
 211 }
 212 
 213 void InterpreterMacroAssembler::read_flat_field(Register entry, Register obj) {
 214   call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flat_field), obj, entry);
 215   membar(Assembler::StoreStore);
 216 }
 217 
 218 void InterpreterMacroAssembler::write_flat_field(Register entry, Register field_offset,
 219                                                  Register tmp1, Register tmp2,
 220                                                  Register obj) {
 221   assert_different_registers(entry, field_offset, tmp1, tmp2, obj);
 222   Label slow_path, done;
 223 
 224   load_unsigned_byte(tmp1, Address(entry, in_bytes(ResolvedFieldEntry::flags_offset())));
 225   test_field_is_not_null_free_inline_type(tmp1, noreg /* temp */, slow_path);
 226 
 227   null_check(r0); // FIXME JDK-8341120
 228 
 229   add(obj, obj, field_offset);
 230 
 231   load_klass(tmp1, r0);
 232   payload_address(r0, r0, tmp1);
 233 
 234   Register layout_info = field_offset;
 235   load_unsigned_short(tmp1, Address(entry, in_bytes(ResolvedFieldEntry::field_index_offset())));
 236   ldr(tmp2, Address(entry, in_bytes(ResolvedFieldEntry::field_holder_offset())));
 237   inline_layout_info(tmp2, tmp1, layout_info);
 238 
 239   flat_field_copy(IN_HEAP, r0, obj, layout_info);
 240   b(done);
 241 
 242   bind(slow_path);
 243   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flat_field), obj, r0, entry);
 244   bind(done);
 245 }
 246 
 247 // Load object from cpool->resolved_references(index)
 248 void InterpreterMacroAssembler::load_resolved_reference_at_index(
 249                                            Register result, Register index, Register tmp) {
 250   assert_different_registers(result, index);
 251 
 252   get_constant_pool(result);
 253   // load pointer for resolved_references[] objArray
 254   ldr(result, Address(result, ConstantPool::cache_offset()));
 255   ldr(result, Address(result, ConstantPoolCache::resolved_references_offset()));
 256   resolve_oop_handle(result, tmp, rscratch2);
 257   // Add in the index
 258   add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 259   load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)), tmp, rscratch2);
 260 }
 261 
 262 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
 263                              Register cpool, Register index, Register klass, Register temp) {
 264   add(temp, cpool, index, LSL, LogBytesPerWord);
 265   ldrh(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index
 266   ldr(klass, Address(cpool,  ConstantPool::resolved_klasses_offset())); // klass = cpool->_resolved_klasses
 267   add(klass, klass, temp, LSL, LogBytesPerWord);
 268   ldr(klass, Address(klass, Array<Klass*>::base_offset_in_bytes()));
 269 }
 270 
 271 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
 272 // subtype of super_klass.
 273 //
 274 // Args:
 275 //      r0: superklass
 276 //      Rsub_klass: subklass
 277 //
 278 // Kills:
 279 //      r2
 280 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
 281                                                   Label& ok_is_subtype,
 282                                                   bool profile) {
 283   assert(Rsub_klass != r0, "r0 holds superklass");
 284   assert(Rsub_klass != r2, "r2 holds 2ndary super array length");
 285 
 286   // Profile the not-null value's klass.
 287   if (profile) {
 288     profile_typecheck(r2, Rsub_klass); // blows r2
 289   }
 290   // Do the check.
 291   check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2
 292 }
 293 
 294 // Java Expression Stack
 295 
 296 void InterpreterMacroAssembler::pop_ptr(Register r) {
 297   ldr(r, post(esp, wordSize));
 298 }
 299 
 300 void InterpreterMacroAssembler::pop_i(Register r) {
 301   ldrw(r, post(esp, wordSize));
 302 }
 303 
 304 void InterpreterMacroAssembler::pop_l(Register r) {
 305   ldr(r, post(esp, 2 * Interpreter::stackElementSize));
 306 }
 307 
 308 void InterpreterMacroAssembler::push_ptr(Register r) {
 309   str(r, pre(esp, -wordSize));
 310  }
 311 
 312 void InterpreterMacroAssembler::push_i(Register r) {
 313   str(r, pre(esp, -wordSize));
 314 }
 315 
 316 void InterpreterMacroAssembler::push_l(Register r) {
 317   str(zr, pre(esp, -wordSize));
 318   str(r, pre(esp, - wordSize));
 319 }
 320 
 321 void InterpreterMacroAssembler::pop_f(FloatRegister r) {
 322   ldrs(r, post(esp, wordSize));
 323 }
 324 
 325 void InterpreterMacroAssembler::pop_d(FloatRegister r) {
 326   ldrd(r, post(esp, 2 * Interpreter::stackElementSize));
 327 }
 328 
 329 void InterpreterMacroAssembler::push_f(FloatRegister r) {
 330   strs(r, pre(esp, -wordSize));
 331 }
 332 
 333 void InterpreterMacroAssembler::push_d(FloatRegister r) {
 334   strd(r, pre(esp, 2* -wordSize));
 335 }
 336 
 337 void InterpreterMacroAssembler::pop(TosState state) {
 338   switch (state) {
 339   case atos: pop_ptr();                 break;
 340   case btos:
 341   case ztos:
 342   case ctos:
 343   case stos:
 344   case itos: pop_i();                   break;
 345   case ltos: pop_l();                   break;
 346   case ftos: pop_f();                   break;
 347   case dtos: pop_d();                   break;
 348   case vtos: /* nothing to do */        break;
 349   default:   ShouldNotReachHere();
 350   }
 351   interp_verify_oop(r0, state);
 352 }
 353 
 354 void InterpreterMacroAssembler::push(TosState state) {
 355   interp_verify_oop(r0, state);
 356   switch (state) {
 357   case atos: push_ptr();                break;
 358   case btos:
 359   case ztos:
 360   case ctos:
 361   case stos:
 362   case itos: push_i();                  break;
 363   case ltos: push_l();                  break;
 364   case ftos: push_f();                  break;
 365   case dtos: push_d();                  break;
 366   case vtos: /* nothing to do */        break;
 367   default  : ShouldNotReachHere();
 368   }
 369 }
 370 
 371 // Helpers for swap and dup
 372 void InterpreterMacroAssembler::load_ptr(int n, Register val) {
 373   ldr(val, Address(esp, Interpreter::expr_offset_in_bytes(n)));
 374 }
 375 
 376 void InterpreterMacroAssembler::store_ptr(int n, Register val) {
 377   str(val, Address(esp, Interpreter::expr_offset_in_bytes(n)));
 378 }
 379 
 380 void InterpreterMacroAssembler::load_float(Address src) {
 381   ldrs(v0, src);
 382 }
 383 
 384 void InterpreterMacroAssembler::load_double(Address src) {
 385   ldrd(v0, src);
 386 }
 387 
 388 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() {
 389   // set sender sp
 390   mov(r19_sender_sp, sp);
 391   // record last_sp
 392   sub(rscratch1, esp, rfp);
 393   asr(rscratch1, rscratch1, Interpreter::logStackElementSize);
 394   str(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
 395 }
 396 
 397 // Jump to from_interpreted entry of a call unless single stepping is possible
 398 // in this thread in which case we must call the i2i entry
 399 void InterpreterMacroAssembler::jump_from_interpreted(Register method, Register temp) {
 400   prepare_to_jump_from_interpreted();
 401 
 402   if (JvmtiExport::can_post_interpreter_events()) {
 403     Label run_compiled_code;
 404     // JVMTI events, such as single-stepping, are implemented partly by avoiding running
 405     // compiled code in threads for which the event is enabled.  Check here for
 406     // interp_only_mode if these events CAN be enabled.
 407     ldrw(rscratch1, Address(rthread, JavaThread::interp_only_mode_offset()));
 408     cbzw(rscratch1, run_compiled_code);
 409     ldr(rscratch1, Address(method, Method::interpreter_entry_offset()));
 410     br(rscratch1);
 411     bind(run_compiled_code);
 412   }
 413 
 414   ldr(rscratch1, Address(method, Method::from_interpreted_offset()));
 415   br(rscratch1);
 416 }
 417 
 418 // The following two routines provide a hook so that an implementation
 419 // can schedule the dispatch in two parts.  amd64 does not do this.
 420 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) {
 421 }
 422 
 423 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {
 424     dispatch_next(state, step);
 425 }
 426 
 427 void InterpreterMacroAssembler::dispatch_base(TosState state,
 428                                               address* table,
 429                                               bool verifyoop,
 430                                               bool generate_poll) {
 431   if (VerifyActivationFrameSize) {
 432     Label L;
 433     sub(rscratch2, rfp, esp);
 434     int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize;
 435     subs(rscratch2, rscratch2, min_frame_size);
 436     br(Assembler::GE, L);
 437     stop("broken stack frame");
 438     bind(L);
 439   }
 440   if (verifyoop) {
 441     interp_verify_oop(r0, state);
 442   }
 443 
 444   Label safepoint;
 445   address* const safepoint_table = Interpreter::safept_table(state);
 446   bool needs_thread_local_poll = generate_poll && table != safepoint_table;
 447 
 448   if (needs_thread_local_poll) {
 449     NOT_PRODUCT(block_comment("Thread-local Safepoint poll"));
 450     ldr(rscratch2, Address(rthread, JavaThread::polling_word_offset()));
 451     tbnz(rscratch2, exact_log2(SafepointMechanism::poll_bit()), safepoint);
 452   }
 453 
 454   if (table == Interpreter::dispatch_table(state)) {
 455     addw(rscratch2, rscratch1, Interpreter::distance_from_dispatch_table(state));
 456     ldr(rscratch2, Address(rdispatch, rscratch2, Address::uxtw(3)));
 457   } else {
 458     mov(rscratch2, (address)table);
 459     ldr(rscratch2, Address(rscratch2, rscratch1, Address::uxtw(3)));
 460   }
 461   br(rscratch2);
 462 
 463   if (needs_thread_local_poll) {
 464     bind(safepoint);
 465     lea(rscratch2, ExternalAddress((address)safepoint_table));
 466     ldr(rscratch2, Address(rscratch2, rscratch1, Address::uxtw(3)));
 467     br(rscratch2);
 468   }
 469 }
 470 
 471 void InterpreterMacroAssembler::dispatch_only(TosState state, bool generate_poll) {
 472   dispatch_base(state, Interpreter::dispatch_table(state), true, generate_poll);
 473 }
 474 
 475 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) {
 476   dispatch_base(state, Interpreter::normal_table(state));
 477 }
 478 
 479 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
 480   dispatch_base(state, Interpreter::normal_table(state), false);
 481 }
 482 
 483 
 484 void InterpreterMacroAssembler::dispatch_next(TosState state, int step, bool generate_poll) {
 485   // load next bytecode
 486   ldrb(rscratch1, Address(pre(rbcp, step)));
 487   dispatch_base(state, Interpreter::dispatch_table(state), /*verifyoop*/true, generate_poll);
 488 }
 489 
 490 void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) {
 491   // load current bytecode
 492   ldrb(rscratch1, Address(rbcp, 0));
 493   dispatch_base(state, table);
 494 }
 495 
 496 // remove activation
 497 //
 498 // Unlock the receiver if this is a synchronized method.
 499 // Unlock any Java monitors from synchronized blocks.
 500 // Apply stack watermark barrier.
 501 // Notify JVMTI.
 502 // Remove the activation from the stack.
 503 //
 504 // If there are locked Java monitors
 505 //    If throw_monitor_exception
 506 //       throws IllegalMonitorStateException
 507 //    Else if install_monitor_exception
 508 //       installs IllegalMonitorStateException
 509 //    Else
 510 //       no error processing
 511 void InterpreterMacroAssembler::remove_activation(TosState state,
 512                                                   bool throw_monitor_exception,
 513                                                   bool install_monitor_exception,
 514                                                   bool notify_jvmdi) {
 515   // Note: Registers r3 xmm0 may be in use for the
 516   // result check if synchronized method
 517   Label unlocked, unlock, no_unlock;
 518 
 519 #ifdef ASSERT
 520   Label not_preempted;
 521   ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset()));
 522   cbz(rscratch1, not_preempted);
 523   stop("remove_activation: should not have alternate return address set");
 524   bind(not_preempted);
 525 #endif /* ASSERT */
 526 
 527   // get the value of _do_not_unlock_if_synchronized into r3
 528   const Address do_not_unlock_if_synchronized(rthread,
 529     in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
 530   ldrb(r3, do_not_unlock_if_synchronized);
 531   strb(zr, do_not_unlock_if_synchronized); // reset the flag
 532 
 533  // get method access flags
 534   ldr(r1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
 535   ldrh(r2, Address(r1, Method::access_flags_offset()));
 536   tbz(r2, exact_log2(JVM_ACC_SYNCHRONIZED), unlocked);
 537 
 538   // Don't unlock anything if the _do_not_unlock_if_synchronized flag
 539   // is set.
 540   cbnz(r3, no_unlock);
 541 
 542   // unlock monitor
 543   push(state); // save result
 544 
 545   // BasicObjectLock will be first in list, since this is a
 546   // synchronized method. However, need to check that the object has
 547   // not been unlocked by an explicit monitorexit bytecode.
 548   const Address monitor(rfp, frame::interpreter_frame_initial_sp_offset *
 549                         wordSize - (int) sizeof(BasicObjectLock));
 550   // We use c_rarg1 so that if we go slow path it will be the correct
 551   // register for unlock_object to pass to VM directly
 552   lea(c_rarg1, monitor); // address of first monitor
 553 
 554   ldr(r0, Address(c_rarg1, BasicObjectLock::obj_offset()));
 555   cbnz(r0, unlock);
 556 
 557   pop(state);
 558   if (throw_monitor_exception) {
 559     // Entry already unlocked, need to throw exception
 560     call_VM(noreg, CAST_FROM_FN_PTR(address,
 561                    InterpreterRuntime::throw_illegal_monitor_state_exception));
 562     should_not_reach_here();
 563   } else {
 564     // Monitor already unlocked during a stack unroll. If requested,
 565     // install an illegal_monitor_state_exception.  Continue with
 566     // stack unrolling.
 567     if (install_monitor_exception) {
 568       call_VM(noreg, CAST_FROM_FN_PTR(address,
 569                      InterpreterRuntime::new_illegal_monitor_state_exception));
 570     }
 571     b(unlocked);
 572   }
 573 
 574   bind(unlock);
 575   unlock_object(c_rarg1);
 576   pop(state);
 577 
 578   // Check that for block-structured locking (i.e., that all locked
 579   // objects has been unlocked)
 580   bind(unlocked);
 581 
 582   // r0: Might contain return value
 583 
 584   // Check that all monitors are unlocked
 585   {
 586     Label loop, exception, entry, restart;
 587     const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
 588     const Address monitor_block_top(
 589         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
 590     const Address monitor_block_bot(
 591         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
 592 
 593     bind(restart);
 594     // We use c_rarg1 so that if we go slow path it will be the correct
 595     // register for unlock_object to pass to VM directly
 596     ldr(c_rarg1, monitor_block_top); // derelativize pointer
 597     lea(c_rarg1, Address(rfp, c_rarg1, Address::lsl(Interpreter::logStackElementSize)));
 598     // c_rarg1 points to current entry, starting with top-most entry
 599 
 600     lea(r19, monitor_block_bot);  // points to word before bottom of
 601                                   // monitor block
 602     b(entry);
 603 
 604     // Entry already locked, need to throw exception
 605     bind(exception);
 606 
 607     if (throw_monitor_exception) {
 608       // Throw exception
 609       MacroAssembler::call_VM(noreg,
 610                               CAST_FROM_FN_PTR(address, InterpreterRuntime::
 611                                    throw_illegal_monitor_state_exception));
 612       should_not_reach_here();
 613     } else {
 614       // Stack unrolling. Unlock object and install illegal_monitor_exception.
 615       // Unlock does not block, so don't have to worry about the frame.
 616       // We don't have to preserve c_rarg1 since we are going to throw an exception.
 617 
 618       push(state);
 619       unlock_object(c_rarg1);
 620       pop(state);
 621 
 622       if (install_monitor_exception) {
 623         call_VM(noreg, CAST_FROM_FN_PTR(address,
 624                                         InterpreterRuntime::
 625                                         new_illegal_monitor_state_exception));
 626       }
 627 
 628       b(restart);
 629     }
 630 
 631     bind(loop);
 632     // check if current entry is used
 633     ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset()));
 634     cbnz(rscratch1, exception);
 635 
 636     add(c_rarg1, c_rarg1, entry_size); // otherwise advance to next entry
 637     bind(entry);
 638     cmp(c_rarg1, r19); // check if bottom reached
 639     br(Assembler::NE, loop); // if not at bottom then check this entry
 640   }
 641 
 642   bind(no_unlock);
 643 
 644   JFR_ONLY(enter_jfr_critical_section();)
 645 
 646   // The below poll is for the stack watermark barrier. It allows fixing up frames lazily,
 647   // that would normally not be safe to use. Such bad returns into unsafe territory of
 648   // the stack, will call InterpreterRuntime::at_unwind.
 649   Label slow_path;
 650   Label fast_path;
 651   safepoint_poll(slow_path, true /* at_return */, false /* in_nmethod */);
 652   br(Assembler::AL, fast_path);
 653   bind(slow_path);
 654   push(state);
 655   set_last_Java_frame(esp, rfp, pc(), rscratch1);
 656   super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
 657   reset_last_Java_frame(true);
 658   pop(state);
 659   bind(fast_path);
 660 
 661   // JVMTI support. Make sure the safepoint poll test is issued prior.
 662   if (notify_jvmdi) {
 663     notify_method_exit(state, NotifyJVMTI);    // preserve TOSCA
 664   } else {
 665     notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
 666   }
 667 
 668   // remove activation
 669   // get sender esp
 670   ldr(rscratch2,
 671       Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
 672   if (StackReservedPages > 0) {
 673     // testing if reserved zone needs to be re-enabled
 674     Label no_reserved_zone_enabling;
 675 
 676     // check if already enabled - if so no re-enabling needed
 677     assert(sizeof(StackOverflow::StackGuardState) == 4, "unexpected size");
 678     ldrw(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
 679     cmpw(rscratch1, (u1)StackOverflow::stack_guard_enabled);
 680     br(Assembler::EQ, no_reserved_zone_enabling);
 681 
 682     // look for an overflow into the stack reserved zone, i.e.
 683     // interpreter_frame_sender_sp <= JavaThread::reserved_stack_activation
 684     ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
 685     cmp(rscratch2, rscratch1);
 686     br(Assembler::LS, no_reserved_zone_enabling);
 687 
 688     JFR_ONLY(leave_jfr_critical_section();)
 689 
 690     call_VM_leaf(
 691       CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
 692     call_VM(noreg, CAST_FROM_FN_PTR(address,
 693                    InterpreterRuntime::throw_delayed_StackOverflowError));
 694     should_not_reach_here();
 695 
 696     bind(no_reserved_zone_enabling);
 697   }
 698 
 699   if (state == atos && InlineTypeReturnedAsFields) {
 700     Label skip;
 701     Label not_null;
 702     cbnz(r0, not_null);
 703     // Returned value is null, zero all return registers because they may belong to oop fields
 704     mov(j_rarg1, zr);
 705     mov(j_rarg2, zr);
 706     mov(j_rarg3, zr);
 707     mov(j_rarg4, zr);
 708     mov(j_rarg5, zr);
 709     mov(j_rarg6, zr);
 710     mov(j_rarg7, zr);
 711     b(skip);
 712     bind(not_null);
 713 
 714     // Check if we are returning an non-null inline type and load its fields into registers
 715     test_oop_is_not_inline_type(r0, rscratch2, skip, /* can_be_null= */ false);
 716 
 717     // Load fields from a buffered value with an inline class specific handler
 718     load_klass(rscratch1 /*dst*/, r0 /*src*/);
 719     ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
 720     ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
 721     // Unpack handler can be null if inline type is not scalarizable in returns
 722     cbz(rscratch1, skip);
 723 
 724     blr(rscratch1);
 725     bind(skip);
 726     // Check above kills sender esp in rscratch2. Reload it.
 727     ldr(rscratch2, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
 728   }
 729 
 730   // remove frame anchor
 731   leave();
 732 
 733   JFR_ONLY(leave_jfr_critical_section();)
 734 
 735   // restore sender esp
 736   mov(esp, rscratch2);
 737 
 738   // If we're returning to interpreted code we will shortly be
 739   // adjusting SP to allow some space for ESP.  If we're returning to
 740   // compiled code the saved sender SP was saved in sender_sp, so this
 741   // restores it.
 742   andr(sp, esp, -16);
 743 }
 744 
 745 #if INCLUDE_JFR
 746 void InterpreterMacroAssembler::enter_jfr_critical_section() {
 747   const Address sampling_critical_section(rthread, in_bytes(SAMPLING_CRITICAL_SECTION_OFFSET_JFR));
 748   mov(rscratch1, true);
 749   strb(rscratch1, sampling_critical_section);
 750 }
 751 
 752 void InterpreterMacroAssembler::leave_jfr_critical_section() {
 753   const Address sampling_critical_section(rthread, in_bytes(SAMPLING_CRITICAL_SECTION_OFFSET_JFR));
 754   strb(zr, sampling_critical_section);
 755 }
 756 #endif // INCLUDE_JFR
 757 
 758 // Lock object
 759 //
 760 // Args:
 761 //      c_rarg1: BasicObjectLock to be used for locking
 762 //
 763 // Kills:
 764 //      r0
 765 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, .. (param regs)
 766 //      rscratch1, rscratch2 (scratch regs)
 767 void InterpreterMacroAssembler::lock_object(Register lock_reg)
 768 {
 769   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
 770 
 771   const Register tmp = c_rarg2;
 772   const Register obj_reg = c_rarg3; // Will contain the oop
 773   const Register tmp2 = c_rarg4;
 774   const Register tmp3 = c_rarg5;
 775 
 776   // Load object pointer into obj_reg %c_rarg3
 777   ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset()));
 778 
 779   Label slow_case, done;
 780   fast_lock(lock_reg, obj_reg, tmp, tmp2, tmp3, slow_case);
 781   b(done);
 782 
 783   bind(slow_case);
 784 
 785   // Call the runtime routine for slow case
 786   call_VM_preemptable(noreg,
 787           CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
 788           lock_reg);
 789 
 790   bind(done);
 791 }
 792 
 793 
 794 // Unlocks an object. Used in monitorexit bytecode and
 795 // remove_activation.  Throws an IllegalMonitorException if object is
 796 // not locked by current thread.
 797 //
 798 // Args:
 799 //      c_rarg1: BasicObjectLock for lock
 800 //
 801 // Kills:
 802 //      r0
 803 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
 804 //      rscratch1, rscratch2 (scratch regs)
 805 void InterpreterMacroAssembler::unlock_object(Register lock_reg)
 806 {
 807   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
 808 
 809   const Register swap_reg   = r0;
 810   const Register header_reg = c_rarg2;  // Will contain the old oopMark
 811   const Register obj_reg    = c_rarg3;  // Will contain the oop
 812   const Register tmp_reg    = c_rarg4;  // Temporary used by fast_unlock
 813 
 814   save_bcp(); // Save in case of exception
 815 
 816   // Load oop into obj_reg(%c_rarg3)
 817   ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset()));
 818 
 819   // Free entry
 820   str(zr, Address(lock_reg, BasicObjectLock::obj_offset()));
 821 
 822   Label slow_case, done;
 823   fast_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
 824   b(done);
 825 
 826   bind(slow_case);
 827   // Call the runtime routine for slow case.
 828   str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset())); // restore obj
 829   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
 830   bind(done);
 831   restore_bcp();
 832 }
 833 
 834 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
 835                                                          Label& zero_continue) {
 836   assert(ProfileInterpreter, "must be profiling interpreter");
 837   ldr(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
 838   cbz(mdp, zero_continue);
 839 }
 840 
 841 // Set the method data pointer for the current bcp.
 842 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
 843   assert(ProfileInterpreter, "must be profiling interpreter");
 844   Label set_mdp;
 845   stp(r0, r1, Address(pre(sp, -2 * wordSize)));
 846 
 847   // Test MDO to avoid the call if it is null.
 848   ldr(r0, Address(rmethod, in_bytes(Method::method_data_offset())));
 849   cbz(r0, set_mdp);
 850   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rmethod, rbcp);
 851   // r0: mdi
 852   // mdo is guaranteed to be non-zero here, we checked for it before the call.
 853   ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
 854   lea(r1, Address(r1, in_bytes(MethodData::data_offset())));
 855   add(r0, r1, r0);
 856   str(r0, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
 857   bind(set_mdp);
 858   ldp(r0, r1, Address(post(sp, 2 * wordSize)));
 859 }
 860 
 861 void InterpreterMacroAssembler::verify_method_data_pointer() {
 862   assert(ProfileInterpreter, "must be profiling interpreter");
 863 #ifdef ASSERT
 864   Label verify_continue;
 865   stp(r0, r1, Address(pre(sp, -2 * wordSize)));
 866   stp(r2, r3, Address(pre(sp, -2 * wordSize)));
 867   test_method_data_pointer(r3, verify_continue); // If mdp is zero, continue
 868   get_method(r1);
 869 
 870   // If the mdp is valid, it will point to a DataLayout header which is
 871   // consistent with the bcp.  The converse is highly probable also.
 872   ldrsh(r2, Address(r3, in_bytes(DataLayout::bci_offset())));
 873   ldr(rscratch1, Address(r1, Method::const_offset()));
 874   add(r2, r2, rscratch1, Assembler::LSL);
 875   lea(r2, Address(r2, ConstMethod::codes_offset()));
 876   cmp(r2, rbcp);
 877   br(Assembler::EQ, verify_continue);
 878   // r1: method
 879   // rbcp: bcp // rbcp == 22
 880   // r3: mdp
 881   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp),
 882                r1, rbcp, r3);
 883   bind(verify_continue);
 884   ldp(r2, r3, Address(post(sp, 2 * wordSize)));
 885   ldp(r0, r1, Address(post(sp, 2 * wordSize)));
 886 #endif // ASSERT
 887 }
 888 
 889 
 890 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in,
 891                                                 int constant,
 892                                                 Register value) {
 893   assert(ProfileInterpreter, "must be profiling interpreter");
 894   Address data(mdp_in, constant);
 895   str(value, data);
 896 }
 897 
 898 
 899 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
 900                                                       int constant) {
 901   increment_mdp_data_at(mdp_in, noreg, constant);
 902 }
 903 
 904 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
 905                                                       Register index,
 906                                                       int constant) {
 907   assert(ProfileInterpreter, "must be profiling interpreter");
 908 
 909   assert_different_registers(rscratch2, rscratch1, mdp_in, index);
 910 
 911   Address addr1(mdp_in, constant);
 912   Address addr2(rscratch2, index, Address::lsl(0));
 913   Address &addr = addr1;
 914   if (index != noreg) {
 915     lea(rscratch2, addr1);
 916     addr = addr2;
 917   }
 918 
 919   increment(addr, DataLayout::counter_increment);
 920 }
 921 
 922 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in,
 923                                                 int flag_byte_constant) {
 924   assert(ProfileInterpreter, "must be profiling interpreter");
 925   int flags_offset = in_bytes(DataLayout::flags_offset());
 926   // Set the flag
 927   ldrb(rscratch1, Address(mdp_in, flags_offset));
 928   orr(rscratch1, rscratch1, flag_byte_constant);
 929   strb(rscratch1, Address(mdp_in, flags_offset));
 930 }
 931 
 932 
 933 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in,
 934                                                  int offset,
 935                                                  Register value,
 936                                                  Register test_value_out,
 937                                                  Label& not_equal_continue) {
 938   assert(ProfileInterpreter, "must be profiling interpreter");
 939   if (test_value_out == noreg) {
 940     ldr(rscratch1, Address(mdp_in, offset));
 941     cmp(value, rscratch1);
 942   } else {
 943     // Put the test value into a register, so caller can use it:
 944     ldr(test_value_out, Address(mdp_in, offset));
 945     cmp(value, test_value_out);
 946   }
 947   br(Assembler::NE, not_equal_continue);
 948 }
 949 
 950 
 951 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in,
 952                                                      int offset_of_disp) {
 953   assert(ProfileInterpreter, "must be profiling interpreter");
 954   ldr(rscratch1, Address(mdp_in, offset_of_disp));
 955   add(mdp_in, mdp_in, rscratch1, LSL);
 956   str(mdp_in, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
 957 }
 958 
 959 
 960 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in,
 961                                                      Register reg,
 962                                                      int offset_of_disp) {
 963   assert(ProfileInterpreter, "must be profiling interpreter");
 964   lea(rscratch1, Address(mdp_in, offset_of_disp));
 965   ldr(rscratch1, Address(rscratch1, reg, Address::lsl(0)));
 966   add(mdp_in, mdp_in, rscratch1, LSL);
 967   str(mdp_in, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
 968 }
 969 
 970 
 971 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in,
 972                                                        int constant) {
 973   assert(ProfileInterpreter, "must be profiling interpreter");
 974   add(mdp_in, mdp_in, (unsigned)constant);
 975   str(mdp_in, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
 976 }
 977 
 978 
 979 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {
 980   assert(ProfileInterpreter, "must be profiling interpreter");
 981   // save/restore across call_VM
 982   stp(zr, return_bci, Address(pre(sp, -2 * wordSize)));
 983   call_VM(noreg,
 984           CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret),
 985           return_bci);
 986   ldp(zr, return_bci, Address(post(sp, 2 * wordSize)));
 987 }
 988 
 989 
 990 void InterpreterMacroAssembler::profile_taken_branch(Register mdp) {
 991   if (ProfileInterpreter) {
 992     Label profile_continue;
 993 
 994     // If no method data exists, go to profile_continue.
 995     test_method_data_pointer(mdp, profile_continue);
 996 
 997     // We are taking a branch.  Increment the taken count.
 998     increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
 999 
1000     // The method data pointer needs to be updated to reflect the new target.
1001     update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
1002     bind(profile_continue);
1003   }
1004 }
1005 
1006 
1007 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp, bool acmp) {
1008   if (ProfileInterpreter) {
1009     Label profile_continue;
1010 
1011     // If no method data exists, go to profile_continue.
1012     test_method_data_pointer(mdp, profile_continue);
1013 
1014     // We are not taking a branch.  Increment the not taken count.
1015     increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
1016 
1017     // The method data pointer needs to be updated to correspond to
1018     // the next bytecode
1019     update_mdp_by_constant(mdp, acmp ? in_bytes(ACmpData::acmp_data_size()) : in_bytes(BranchData::branch_data_size()));
1020     bind(profile_continue);
1021   }
1022 }
1023 
1024 
1025 void InterpreterMacroAssembler::profile_call(Register mdp) {
1026   if (ProfileInterpreter) {
1027     Label profile_continue;
1028 
1029     // If no method data exists, go to profile_continue.
1030     test_method_data_pointer(mdp, profile_continue);
1031 
1032     // We are making a call.  Increment the count.
1033     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1034 
1035     // The method data pointer needs to be updated to reflect the new target.
1036     update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1037     bind(profile_continue);
1038   }
1039 }
1040 
1041 void InterpreterMacroAssembler::profile_final_call(Register mdp) {
1042   if (ProfileInterpreter) {
1043     Label profile_continue;
1044 
1045     // If no method data exists, go to profile_continue.
1046     test_method_data_pointer(mdp, profile_continue);
1047 
1048     // We are making a call.  Increment the count.
1049     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1050 
1051     // The method data pointer needs to be updated to reflect the new target.
1052     update_mdp_by_constant(mdp,
1053                            in_bytes(VirtualCallData::
1054                                     virtual_call_data_size()));
1055     bind(profile_continue);
1056   }
1057 }
1058 
1059 
1060 void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
1061                                                      Register mdp) {
1062   if (ProfileInterpreter) {
1063     Label profile_continue;
1064 
1065     // If no method data exists, go to profile_continue.
1066     test_method_data_pointer(mdp, profile_continue);
1067 
1068     // Record the receiver type.
1069     profile_receiver_type(receiver, mdp, 0);
1070 
1071     // The method data pointer needs to be updated to reflect the new target.
1072     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1073     bind(profile_continue);
1074   }
1075 }
1076 
1077 void InterpreterMacroAssembler::profile_ret(Register return_bci,
1078                                             Register mdp) {
1079   if (ProfileInterpreter) {
1080     Label profile_continue;
1081     uint row;
1082 
1083     // If no method data exists, go to profile_continue.
1084     test_method_data_pointer(mdp, profile_continue);
1085 
1086     // Update the total ret count.
1087     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1088 
1089     for (row = 0; row < RetData::row_limit(); row++) {
1090       Label next_test;
1091 
1092       // See if return_bci is equal to bci[n]:
1093       test_mdp_data_at(mdp,
1094                        in_bytes(RetData::bci_offset(row)),
1095                        return_bci, noreg,
1096                        next_test);
1097 
1098       // return_bci is equal to bci[n].  Increment the count.
1099       increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)));
1100 
1101       // The method data pointer needs to be updated to reflect the new target.
1102       update_mdp_by_offset(mdp,
1103                            in_bytes(RetData::bci_displacement_offset(row)));
1104       b(profile_continue);
1105       bind(next_test);
1106     }
1107 
1108     update_mdp_for_ret(return_bci);
1109 
1110     bind(profile_continue);
1111   }
1112 }
1113 
1114 void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
1115   if (ProfileInterpreter) {
1116     Label profile_continue;
1117 
1118     // If no method data exists, go to profile_continue.
1119     test_method_data_pointer(mdp, profile_continue);
1120 
1121     set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1122 
1123     // The method data pointer needs to be updated.
1124     int mdp_delta = in_bytes(BitData::bit_data_size());
1125     if (TypeProfileCasts) {
1126       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1127     }
1128     update_mdp_by_constant(mdp, mdp_delta);
1129 
1130     bind(profile_continue);
1131   }
1132 }
1133 
1134 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass) {
1135   if (ProfileInterpreter) {
1136     Label profile_continue;
1137 
1138     // If no method data exists, go to profile_continue.
1139     test_method_data_pointer(mdp, profile_continue);
1140 
1141     // The method data pointer needs to be updated.
1142     int mdp_delta = in_bytes(BitData::bit_data_size());
1143     if (TypeProfileCasts) {
1144       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1145 
1146       // Record the object type.
1147       profile_receiver_type(klass, mdp, 0);
1148     }
1149     update_mdp_by_constant(mdp, mdp_delta);
1150 
1151     bind(profile_continue);
1152   }
1153 }
1154 
1155 void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
1156   if (ProfileInterpreter) {
1157     Label profile_continue;
1158 
1159     // If no method data exists, go to profile_continue.
1160     test_method_data_pointer(mdp, profile_continue);
1161 
1162     // Update the default case count
1163     increment_mdp_data_at(mdp,
1164                           in_bytes(MultiBranchData::default_count_offset()));
1165 
1166     // The method data pointer needs to be updated.
1167     update_mdp_by_offset(mdp,
1168                          in_bytes(MultiBranchData::
1169                                   default_displacement_offset()));
1170 
1171     bind(profile_continue);
1172   }
1173 }
1174 
1175 void InterpreterMacroAssembler::profile_switch_case(Register index,
1176                                                     Register mdp,
1177                                                     Register reg2) {
1178   if (ProfileInterpreter) {
1179     Label profile_continue;
1180 
1181     // If no method data exists, go to profile_continue.
1182     test_method_data_pointer(mdp, profile_continue);
1183 
1184     // Build the base (index * per_case_size_in_bytes()) +
1185     // case_array_offset_in_bytes()
1186     movw(reg2, in_bytes(MultiBranchData::per_case_size()));
1187     movw(rscratch1, in_bytes(MultiBranchData::case_array_offset()));
1188     Assembler::maddw(index, index, reg2, rscratch1);
1189 
1190     // Update the case count
1191     increment_mdp_data_at(mdp,
1192                           index,
1193                           in_bytes(MultiBranchData::relative_count_offset()));
1194 
1195     // The method data pointer needs to be updated.
1196     update_mdp_by_offset(mdp,
1197                          index,
1198                          in_bytes(MultiBranchData::
1199                                   relative_displacement_offset()));
1200 
1201     bind(profile_continue);
1202   }
1203 }
1204 
1205 template <class ArrayData> void InterpreterMacroAssembler::profile_array_type(Register mdp,
1206                                                                               Register array,
1207                                                                               Register tmp) {
1208   if (ProfileInterpreter) {
1209     Label profile_continue;
1210 
1211     // If no method data exists, go to profile_continue.
1212     test_method_data_pointer(mdp, profile_continue);
1213 
1214     mov(tmp, array);
1215     profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset())));
1216 
1217     Label not_flat;
1218     test_non_flat_array_oop(array, tmp, not_flat);
1219 
1220     set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant());
1221 
1222     bind(not_flat);
1223 
1224     Label not_null_free;
1225     test_non_null_free_array_oop(array, tmp, not_null_free);
1226 
1227     set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant());
1228 
1229     bind(not_null_free);
1230 
1231     bind(profile_continue);
1232   }
1233 }
1234 
1235 template void InterpreterMacroAssembler::profile_array_type<ArrayLoadData>(Register mdp,
1236                                                                            Register array,
1237                                                                            Register tmp);
1238 template void InterpreterMacroAssembler::profile_array_type<ArrayStoreData>(Register mdp,
1239                                                                             Register array,
1240                                                                             Register tmp);
1241 
1242 void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) {
1243   if (ProfileInterpreter) {
1244     Label profile_continue;
1245 
1246     // If no method data exists, go to profile_continue.
1247     test_method_data_pointer(mdp, profile_continue);
1248 
1249     Label done, update;
1250     cbnz(element, update);
1251     set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1252     b(done);
1253 
1254     bind(update);
1255     load_klass(tmp, element);
1256 
1257     // Record the object type.
1258     profile_receiver_type(tmp, mdp, 0);
1259 
1260     bind(done);
1261 
1262     // The method data pointer needs to be updated.
1263     update_mdp_by_constant(mdp, in_bytes(ArrayStoreData::array_store_data_size()));
1264 
1265     bind(profile_continue);
1266   }
1267 }
1268 
1269 
1270 void InterpreterMacroAssembler::profile_element_type(Register mdp,
1271                                                      Register element,
1272                                                      Register tmp) {
1273   if (ProfileInterpreter) {
1274     Label profile_continue;
1275 
1276     // If no method data exists, go to profile_continue.
1277     test_method_data_pointer(mdp, profile_continue);
1278 
1279     mov(tmp, element);
1280     profile_obj_type(tmp, Address(mdp, in_bytes(ArrayLoadData::element_offset())));
1281 
1282     // The method data pointer needs to be updated.
1283     update_mdp_by_constant(mdp, in_bytes(ArrayLoadData::array_load_data_size()));
1284 
1285     bind(profile_continue);
1286   }
1287 }
1288 
1289 void InterpreterMacroAssembler::profile_acmp(Register mdp,
1290                                              Register left,
1291                                              Register right,
1292                                              Register tmp) {
1293   if (ProfileInterpreter) {
1294     Label profile_continue;
1295 
1296     // If no method data exists, go to profile_continue.
1297     test_method_data_pointer(mdp, profile_continue);
1298 
1299     mov(tmp, left);
1300     profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::left_offset())));
1301 
1302     Label left_not_inline_type;
1303     test_oop_is_not_inline_type(left, tmp, left_not_inline_type);
1304     set_mdp_flag_at(mdp, ACmpData::left_inline_type_byte_constant());
1305     bind(left_not_inline_type);
1306 
1307     mov(tmp, right);
1308     profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::right_offset())));
1309 
1310     Label right_not_inline_type;
1311     test_oop_is_not_inline_type(right, tmp, right_not_inline_type);
1312     set_mdp_flag_at(mdp, ACmpData::right_inline_type_byte_constant());
1313     bind(right_not_inline_type);
1314 
1315     bind(profile_continue);
1316   }
1317 }
1318 
1319 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1320   if (state == atos) {
1321     MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1322   }
1323 }
1324 
1325 void InterpreterMacroAssembler::notify_method_entry() {
1326   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1327   // track stack depth.  If it is possible to enter interp_only_mode we add
1328   // the code to check if the event should be sent.
1329   if (JvmtiExport::can_post_interpreter_events()) {
1330     Label L;
1331     ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1332     cbzw(r3, L);
1333     call_VM(noreg, CAST_FROM_FN_PTR(address,
1334                                     InterpreterRuntime::post_method_entry));
1335     bind(L);
1336   }
1337 
1338   if (DTraceMethodProbes) {
1339     get_method(c_rarg1);
1340     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
1341                  rthread, c_rarg1);
1342   }
1343 
1344   // RedefineClasses() tracing support for obsolete method entry
1345   if (log_is_enabled(Trace, redefine, class, obsolete)) {
1346     get_method(c_rarg1);
1347     call_VM_leaf(
1348       CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
1349       rthread, c_rarg1);
1350   }
1351 
1352  }
1353 
1354 
1355 void InterpreterMacroAssembler::notify_method_exit(
1356     TosState state, NotifyMethodExitMode mode) {
1357   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1358   // track stack depth.  If it is possible to enter interp_only_mode we add
1359   // the code to check if the event should be sent.
1360   if (mode == NotifyJVMTI && JvmtiExport::can_post_interpreter_events()) {
1361     Label L;
1362     // Note: frame::interpreter_frame_result has a dependency on how the
1363     // method result is saved across the call to post_method_exit. If this
1364     // is changed then the interpreter_frame_result implementation will
1365     // need to be updated too.
1366 
1367     // template interpreter will leave the result on the top of the stack.
1368     push(state);
1369     ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1370     cbz(r3, L);
1371     call_VM(noreg,
1372             CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
1373     bind(L);
1374     pop(state);
1375   }
1376 
1377   if (DTraceMethodProbes) {
1378     push(state);
1379     get_method(c_rarg1);
1380     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
1381                  rthread, c_rarg1);
1382     pop(state);
1383   }
1384 }
1385 
1386 
1387 // Jump if ((*counter_addr += increment) & mask) satisfies the condition.
1388 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
1389                                                         int increment, Address mask,
1390                                                         Register scratch, Register scratch2,
1391                                                         bool preloaded, Condition cond,
1392                                                         Label* where) {
1393   if (!preloaded) {
1394     ldrw(scratch, counter_addr);
1395   }
1396   add(scratch, scratch, increment);
1397   strw(scratch, counter_addr);
1398   ldrw(scratch2, mask);
1399   ands(scratch, scratch, scratch2);
1400   br(cond, *where);
1401 }
1402 
1403 void InterpreterMacroAssembler::call_VM_leaf_base(address entry_point,
1404                                                   int number_of_arguments) {
1405   // interpreter specific
1406   //
1407   // Note: No need to save/restore rbcp & rlocals pointer since these
1408   //       are callee saved registers and no blocking/ GC can happen
1409   //       in leaf calls.
1410 #ifdef ASSERT
1411   {
1412     Label L;
1413     ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1414     cbz(rscratch1, L);
1415     stop("InterpreterMacroAssembler::call_VM_leaf_base:"
1416          " last_sp != nullptr");
1417     bind(L);
1418   }
1419 #endif /* ASSERT */
1420   // super call
1421   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
1422 }
1423 
1424 void InterpreterMacroAssembler::call_VM_base(Register oop_result,
1425                                              Register java_thread,
1426                                              Register last_java_sp,
1427                                              Label*   return_pc,
1428                                              address  entry_point,
1429                                              int      number_of_arguments,
1430                                              bool     check_exceptions) {
1431   // interpreter specific
1432   //
1433   // Note: Could avoid restoring locals ptr (callee saved) - however doesn't
1434   //       really make a difference for these runtime calls, since they are
1435   //       slow anyway. Btw., bcp must be saved/restored since it may change
1436   //       due to GC.
1437   // assert(java_thread == noreg , "not expecting a precomputed java thread");
1438   save_bcp();
1439 #ifdef ASSERT
1440   {
1441     Label L;
1442     ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1443     cbz(rscratch1, L);
1444     stop("InterpreterMacroAssembler::call_VM_base:"
1445          " last_sp != nullptr");
1446     bind(L);
1447   }
1448 #endif /* ASSERT */
1449   // super call
1450   MacroAssembler::call_VM_base(oop_result, noreg, last_java_sp,
1451                                return_pc, entry_point,
1452                                number_of_arguments, check_exceptions);
1453 // interpreter specific
1454   restore_bcp();
1455   restore_locals();
1456 }
1457 
1458 void InterpreterMacroAssembler::call_VM_preemptable_helper(Register oop_result,
1459                                                            address entry_point,
1460                                                            int number_of_arguments,
1461                                                            bool check_exceptions) {
1462   assert(InterpreterRuntime::is_preemptable_call(entry_point), "VM call not preemptable, should use call_VM()");
1463   Label resume_pc, not_preempted;
1464 
1465 #ifdef ASSERT
1466   {
1467     Label L1, L2;
1468     ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1469     cbz(rscratch1, L1);
1470     stop("call_VM_preemptable_helper: Should not have alternate return address set");
1471     bind(L1);
1472     // We check this counter in patch_return_pc_with_preempt_stub() during freeze.
1473     incrementw(Address(rthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset()));
1474     ldrw(rscratch1, Address(rthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset()));
1475     cmpw(rscratch1, 0);
1476     br(Assembler::GT, L2);
1477     stop("call_VM_preemptable_helper: should be > 0");
1478     bind(L2);
1479   }
1480 #endif /* ASSERT */
1481 
1482   // Force freeze slow path.
1483   push_cont_fastpath();
1484 
1485   // Make VM call. In case of preemption set last_pc to the one we want to resume to.
1486   // Note: call_VM_base will use resume_pc label to set last_Java_pc.
1487   call_VM_base(noreg, noreg, noreg, &resume_pc, entry_point, number_of_arguments, false /*check_exceptions*/);
1488 
1489   pop_cont_fastpath();
1490 
1491 #ifdef ASSERT
1492   {
1493     Label L;
1494     decrementw(Address(rthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset()));
1495     ldrw(rscratch1, Address(rthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset()));
1496     cmpw(rscratch1, 0);
1497     br(Assembler::GE, L);
1498     stop("call_VM_preemptable_helper: should be >= 0");
1499     bind(L);
1500   }
1501 #endif /* ASSERT */
1502 
1503   // Check if preempted.
1504   ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1505   cbz(rscratch1, not_preempted);
1506   str(zr, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1507   br(rscratch1);
1508 
1509   // In case of preemption, this is where we will resume once we finally acquire the monitor.
1510   bind(resume_pc);
1511   restore_after_resume(false /* is_native */);
1512 
1513   bind(not_preempted);
1514   if (check_exceptions) {
1515     // check for pending exceptions
1516     ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1517     Label ok;
1518     cbz(rscratch1, ok);
1519     lea(rscratch1, RuntimeAddress(StubRoutines::forward_exception_entry()));
1520     br(rscratch1);
1521     bind(ok);
1522   }
1523 
1524   // get oop result if there is one and reset the value in the thread
1525   if (oop_result->is_valid()) {
1526     get_vm_result_oop(oop_result, rthread);
1527   }
1528 }
1529 
1530 static void pass_arg1(MacroAssembler* masm, Register arg) {
1531   if (c_rarg1 != arg ) {
1532     masm->mov(c_rarg1, arg);
1533   }
1534 }
1535 
1536 static void pass_arg2(MacroAssembler* masm, Register arg) {
1537   if (c_rarg2 != arg ) {
1538     masm->mov(c_rarg2, arg);
1539   }
1540 }
1541 
1542 void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result,
1543                                          address entry_point,
1544                                          Register arg_1,
1545                                          bool check_exceptions) {
1546   pass_arg1(this, arg_1);
1547   call_VM_preemptable_helper(oop_result, entry_point, 1, check_exceptions);
1548 }
1549 
1550 void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result,
1551                                          address entry_point,
1552                                          Register arg_1,
1553                                          Register arg_2,
1554                                          bool check_exceptions) {
1555   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
1556   pass_arg2(this, arg_2);
1557   pass_arg1(this, arg_1);
1558   call_VM_preemptable_helper(oop_result, entry_point, 2, check_exceptions);
1559 }
1560 
1561 void InterpreterMacroAssembler::restore_after_resume(bool is_native) {
1562   lea(rscratch1, ExternalAddress(Interpreter::cont_resume_interpreter_adapter()));
1563   blr(rscratch1);
1564   if (is_native) {
1565     // On resume we need to set up stack as expected
1566     push(dtos);
1567     push(ltos);
1568   }
1569 }
1570 
1571 void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {
1572   assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
1573   Label update, next, none;
1574 
1575   verify_oop(obj);
1576 
1577   cbnz(obj, update);
1578   orptr(mdo_addr, TypeEntries::null_seen);
1579   b(next);
1580 
1581   bind(update);
1582   load_klass(obj, obj);
1583 
1584   ldr(rscratch1, mdo_addr);
1585   eor(obj, obj, rscratch1);
1586   tst(obj, TypeEntries::type_klass_mask);
1587   br(Assembler::EQ, next); // klass seen before, nothing to
1588                            // do. The unknown bit may have been
1589                            // set already but no need to check.
1590 
1591   tbnz(obj, exact_log2(TypeEntries::type_unknown), next);
1592   // already unknown. Nothing to do anymore.
1593 
1594   cbz(rscratch1, none);
1595   cmp(rscratch1, (u1)TypeEntries::null_seen);
1596   br(Assembler::EQ, none);
1597   // There is a chance that the checks above
1598   // fail if another thread has just set the
1599   // profiling to this obj's klass
1600   eor(obj, obj, rscratch1); // get back original value before XOR
1601   ldr(rscratch1, mdo_addr);
1602   eor(obj, obj, rscratch1);
1603   tst(obj, TypeEntries::type_klass_mask);
1604   br(Assembler::EQ, next);
1605 
1606   // different than before. Cannot keep accurate profile.
1607   orptr(mdo_addr, TypeEntries::type_unknown);
1608   b(next);
1609 
1610   bind(none);
1611   // first time here. Set profile type.
1612   str(obj, mdo_addr);
1613 #ifdef ASSERT
1614   andr(obj, obj, TypeEntries::type_mask);
1615   verify_klass_ptr(obj);
1616 #endif
1617 
1618   bind(next);
1619 }
1620 
1621 void InterpreterMacroAssembler::profile_arguments_type(Register mdp, Register callee, Register tmp, bool is_virtual) {
1622   if (!ProfileInterpreter) {
1623     return;
1624   }
1625 
1626   if (MethodData::profile_arguments() || MethodData::profile_return()) {
1627     Label profile_continue;
1628 
1629     test_method_data_pointer(mdp, profile_continue);
1630 
1631     int off_to_start = is_virtual ? in_bytes(VirtualCallData::virtual_call_data_size()) : in_bytes(CounterData::counter_data_size());
1632 
1633     ldrb(rscratch1, Address(mdp, in_bytes(DataLayout::tag_offset()) - off_to_start));
1634     cmp(rscratch1, u1(is_virtual ? DataLayout::virtual_call_type_data_tag : DataLayout::call_type_data_tag));
1635     br(Assembler::NE, profile_continue);
1636 
1637     if (MethodData::profile_arguments()) {
1638       Label done;
1639       int off_to_args = in_bytes(TypeEntriesAtCall::args_data_offset());
1640 
1641       for (int i = 0; i < TypeProfileArgsLimit; i++) {
1642         if (i > 0 || MethodData::profile_return()) {
1643           // If return value type is profiled we may have no argument to profile
1644           ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1645           sub(tmp, tmp, i*TypeStackSlotEntries::per_arg_count());
1646           cmp(tmp, (u1)TypeStackSlotEntries::per_arg_count());
1647           add(rscratch1, mdp, off_to_args);
1648           br(Assembler::LT, done);
1649         }
1650         ldr(tmp, Address(callee, Method::const_offset()));
1651         load_unsigned_short(tmp, Address(tmp, ConstMethod::size_of_parameters_offset()));
1652         // stack offset o (zero based) from the start of the argument
1653         // list, for n arguments translates into offset n - o - 1 from
1654         // the end of the argument list
1655         ldr(rscratch1, Address(mdp, in_bytes(TypeEntriesAtCall::stack_slot_offset(i))));
1656         sub(tmp, tmp, rscratch1);
1657         sub(tmp, tmp, 1);
1658         Address arg_addr = argument_address(tmp);
1659         ldr(tmp, arg_addr);
1660 
1661         Address mdo_arg_addr(mdp, in_bytes(TypeEntriesAtCall::argument_type_offset(i)));
1662         profile_obj_type(tmp, mdo_arg_addr);
1663 
1664         int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
1665         off_to_args += to_add;
1666       }
1667 
1668       if (MethodData::profile_return()) {
1669         ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1670         sub(tmp, tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
1671       }
1672 
1673       add(rscratch1, mdp, off_to_args);
1674       bind(done);
1675       mov(mdp, rscratch1);
1676 
1677       if (MethodData::profile_return()) {
1678         // We're right after the type profile for the last
1679         // argument. tmp is the number of cells left in the
1680         // CallTypeData/VirtualCallTypeData to reach its end. Non null
1681         // if there's a return to profile.
1682         assert(SingleTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
1683         add(mdp, mdp, tmp, LSL, exact_log2(DataLayout::cell_size));
1684       }
1685       str(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1686     } else {
1687       assert(MethodData::profile_return(), "either profile call args or call ret");
1688       update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
1689     }
1690 
1691     // mdp points right after the end of the
1692     // CallTypeData/VirtualCallTypeData, right after the cells for the
1693     // return value type if there's one
1694 
1695     bind(profile_continue);
1696   }
1697 }
1698 
1699 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
1700   assert_different_registers(mdp, ret, tmp, rbcp);
1701   if (ProfileInterpreter && MethodData::profile_return()) {
1702     Label profile_continue, done;
1703 
1704     test_method_data_pointer(mdp, profile_continue);
1705 
1706     if (MethodData::profile_return_jsr292_only()) {
1707       assert(Method::intrinsic_id_size_in_bytes() == 2, "assuming Method::_intrinsic_id is u2");
1708 
1709       // If we don't profile all invoke bytecodes we must make sure
1710       // it's a bytecode we indeed profile. We can't go back to the
1711       // beginning of the ProfileData we intend to update to check its
1712       // type because we're right after it and we don't known its
1713       // length
1714       Label do_profile;
1715       ldrb(rscratch1, Address(rbcp, 0));
1716       cmp(rscratch1, (u1)Bytecodes::_invokedynamic);
1717       br(Assembler::EQ, do_profile);
1718       cmp(rscratch1, (u1)Bytecodes::_invokehandle);
1719       br(Assembler::EQ, do_profile);
1720       get_method(tmp);
1721       ldrh(rscratch1, Address(tmp, Method::intrinsic_id_offset()));
1722       subs(zr, rscratch1, static_cast<int>(vmIntrinsics::_compiledLambdaForm));
1723       br(Assembler::NE, profile_continue);
1724 
1725       bind(do_profile);
1726     }
1727 
1728     Address mdo_ret_addr(mdp, -in_bytes(SingleTypeEntry::size()));
1729     mov(tmp, ret);
1730     profile_obj_type(tmp, mdo_ret_addr);
1731 
1732     bind(profile_continue);
1733   }
1734 }
1735 
1736 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
1737   assert_different_registers(rscratch1, rscratch2, mdp, tmp1, tmp2);
1738   if (ProfileInterpreter && MethodData::profile_parameters()) {
1739     Label profile_continue, done;
1740 
1741     test_method_data_pointer(mdp, profile_continue);
1742 
1743     // Load the offset of the area within the MDO used for
1744     // parameters. If it's negative we're not profiling any parameters
1745     ldrw(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
1746     tbnz(tmp1, 31, profile_continue);  // i.e. sign bit set
1747 
1748     // Compute a pointer to the area for parameters from the offset
1749     // and move the pointer to the slot for the last
1750     // parameters. Collect profiling from last parameter down.
1751     // mdo start + parameters offset + array length - 1
1752     add(mdp, mdp, tmp1);
1753     ldr(tmp1, Address(mdp, ArrayData::array_len_offset()));
1754     sub(tmp1, tmp1, TypeStackSlotEntries::per_arg_count());
1755 
1756     Label loop;
1757     bind(loop);
1758 
1759     int off_base = in_bytes(ParametersTypeData::stack_slot_offset(0));
1760     int type_base = in_bytes(ParametersTypeData::type_offset(0));
1761     int per_arg_scale = exact_log2(DataLayout::cell_size);
1762     add(rscratch1, mdp, off_base);
1763     add(rscratch2, mdp, type_base);
1764 
1765     Address arg_off(rscratch1, tmp1, Address::lsl(per_arg_scale));
1766     Address arg_type(rscratch2, tmp1, Address::lsl(per_arg_scale));
1767 
1768     // load offset on the stack from the slot for this parameter
1769     ldr(tmp2, arg_off);
1770     neg(tmp2, tmp2);
1771     // read the parameter from the local area
1772     ldr(tmp2, Address(rlocals, tmp2, Address::lsl(Interpreter::logStackElementSize)));
1773 
1774     // profile the parameter
1775     profile_obj_type(tmp2, arg_type);
1776 
1777     // go to next parameter
1778     subs(tmp1, tmp1, TypeStackSlotEntries::per_arg_count());
1779     br(Assembler::GE, loop);
1780 
1781     bind(profile_continue);
1782   }
1783 }
1784 
1785 void InterpreterMacroAssembler::load_resolved_indy_entry(Register cache, Register index) {
1786   // Get index out of bytecode pointer, get_cache_entry_pointer_at_bcp
1787   get_cache_index_at_bcp(index, 1, sizeof(u4));
1788   // Get address of invokedynamic array
1789   ldr(cache, Address(rcpool, in_bytes(ConstantPoolCache::invokedynamic_entries_offset())));
1790   // Scale the index to be the entry index * sizeof(ResolvedIndyEntry)
1791   lsl(index, index, log2i_exact(sizeof(ResolvedIndyEntry)));
1792   add(cache, cache, Array<ResolvedIndyEntry>::base_offset_in_bytes());
1793   lea(cache, Address(cache, index));
1794 }
1795 
1796 void InterpreterMacroAssembler::load_field_entry(Register cache, Register index, int bcp_offset) {
1797   // Get index out of bytecode pointer
1798   get_cache_index_at_bcp(index, bcp_offset, sizeof(u2));
1799   // Take shortcut if the size is a power of 2
1800   if (is_power_of_2(sizeof(ResolvedFieldEntry))) {
1801     lsl(index, index, log2i_exact(sizeof(ResolvedFieldEntry))); // Scale index by power of 2
1802   } else {
1803     mov(cache, sizeof(ResolvedFieldEntry));
1804     mul(index, index, cache); // Scale the index to be the entry index * sizeof(ResolvedFieldEntry)
1805   }
1806   // Get address of field entries array
1807   ldr(cache, Address(rcpool, ConstantPoolCache::field_entries_offset()));
1808   add(cache, cache, Array<ResolvedFieldEntry>::base_offset_in_bytes());
1809   lea(cache, Address(cache, index));
1810   // Prevents stale data from being read after the bytecode is patched to the fast bytecode
1811   membar(MacroAssembler::LoadLoad);
1812 }
1813 
1814 void InterpreterMacroAssembler::load_method_entry(Register cache, Register index, int bcp_offset) {
1815   // Get index out of bytecode pointer
1816   get_cache_index_at_bcp(index, bcp_offset, sizeof(u2));
1817   mov(cache, sizeof(ResolvedMethodEntry));
1818   mul(index, index, cache); // Scale the index to be the entry index * sizeof(ResolvedMethodEntry)
1819 
1820   // Get address of field entries array
1821   ldr(cache, Address(rcpool, ConstantPoolCache::method_entries_offset()));
1822   add(cache, cache, Array<ResolvedMethodEntry>::base_offset_in_bytes());
1823   lea(cache, Address(cache, index));
1824 }
1825 
1826 #ifdef ASSERT
1827 void InterpreterMacroAssembler::verify_field_offset(Register reg) {
1828   // Verify the field offset is not in the header, implicitly checks for 0
1829   Label L;
1830   subs(zr, reg, oopDesc::base_offset_in_bytes());
1831   br(Assembler::GE, L);
1832   stop("bad field offset");
1833   bind(L);
1834 }
1835 #endif
--- EOF ---