1 /*
   2  * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "compiler/compiler_globals.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/barrierSetAssembler.hpp"
  30 #include "interp_masm_aarch64.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "interpreter/interpreterRuntime.hpp"
  33 #include "logging/log.hpp"
  34 #include "oops/arrayOop.hpp"
  35 #include "oops/markWord.hpp"
  36 #include "oops/method.hpp"
  37 #include "oops/methodData.hpp"
  38 #include "oops/resolvedFieldEntry.hpp"
  39 #include "oops/resolvedIndyEntry.hpp"
  40 #include "oops/resolvedMethodEntry.hpp"
  41 #include "prims/jvmtiExport.hpp"
  42 #include "prims/jvmtiThreadState.hpp"
  43 #include "runtime/basicLock.hpp"
  44 #include "runtime/frame.inline.hpp"
  45 #include "runtime/javaThread.hpp"
  46 #include "runtime/safepointMechanism.hpp"
  47 #include "runtime/sharedRuntime.hpp"
  48 #include "utilities/powerOfTwo.hpp"
  49 
  50 void InterpreterMacroAssembler::narrow(Register result) {
  51 
  52   // Get method->_constMethod->_result_type
  53   ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
  54   ldr(rscratch1, Address(rscratch1, Method::const_offset()));
  55   ldrb(rscratch1, Address(rscratch1, ConstMethod::result_type_offset()));
  56 
  57   Label done, notBool, notByte, notChar;
  58 
  59   // common case first
  60   cmpw(rscratch1, T_INT);
  61   br(Assembler::EQ, done);
  62 
  63   // mask integer result to narrower return type.
  64   cmpw(rscratch1, T_BOOLEAN);
  65   br(Assembler::NE, notBool);
  66   andw(result, result, 0x1);
  67   b(done);
  68 
  69   bind(notBool);
  70   cmpw(rscratch1, T_BYTE);
  71   br(Assembler::NE, notByte);
  72   sbfx(result, result, 0, 8);
  73   b(done);
  74 
  75   bind(notByte);
  76   cmpw(rscratch1, T_CHAR);
  77   br(Assembler::NE, notChar);
  78   ubfx(result, result, 0, 16);  // truncate upper 16 bits
  79   b(done);
  80 
  81   bind(notChar);
  82   sbfx(result, result, 0, 16);     // sign-extend short
  83 
  84   // Nothing to do for T_INT
  85   bind(done);
  86 }
  87 
  88 void InterpreterMacroAssembler::jump_to_entry(address entry) {
  89   assert(entry, "Entry must have been generated by now");
  90   b(entry);
  91 }
  92 
  93 void InterpreterMacroAssembler::check_and_handle_popframe(Register java_thread) {
  94   if (JvmtiExport::can_pop_frame()) {
  95     Label L;
  96     // Initiate popframe handling only if it is not already being
  97     // processed.  If the flag has the popframe_processing bit set, it
  98     // means that this code is called *during* popframe handling - we
  99     // don't want to reenter.
 100     // This method is only called just after the call into the vm in
 101     // call_VM_base, so the arg registers are available.
 102     ldrw(rscratch1, Address(rthread, JavaThread::popframe_condition_offset()));
 103     tbz(rscratch1, exact_log2(JavaThread::popframe_pending_bit), L);
 104     tbnz(rscratch1, exact_log2(JavaThread::popframe_processing_bit), L);
 105     // Call Interpreter::remove_activation_preserving_args_entry() to get the
 106     // address of the same-named entrypoint in the generated interpreter code.
 107     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));
 108     br(r0);
 109     bind(L);
 110   }
 111 }
 112 
 113 
 114 void InterpreterMacroAssembler::load_earlyret_value(TosState state) {
 115   ldr(r2, Address(rthread, JavaThread::jvmti_thread_state_offset()));
 116   const Address tos_addr(r2, JvmtiThreadState::earlyret_tos_offset());
 117   const Address oop_addr(r2, JvmtiThreadState::earlyret_oop_offset());
 118   const Address val_addr(r2, JvmtiThreadState::earlyret_value_offset());
 119   switch (state) {
 120     case atos: ldr(r0, oop_addr);
 121                str(zr, oop_addr);
 122                interp_verify_oop(r0, state);        break;
 123     case ltos: ldr(r0, val_addr);                   break;
 124     case btos:                                   // fall through
 125     case ztos:                                   // fall through
 126     case ctos:                                   // fall through
 127     case stos:                                   // fall through
 128     case itos: ldrw(r0, val_addr);                  break;
 129     case ftos: ldrs(v0, val_addr);                  break;
 130     case dtos: ldrd(v0, val_addr);                  break;
 131     case vtos: /* nothing to do */                  break;
 132     default  : ShouldNotReachHere();
 133   }
 134   // Clean up tos value in the thread object
 135   movw(rscratch1, (int) ilgl);
 136   strw(rscratch1, tos_addr);
 137   strw(zr, val_addr);
 138 }
 139 
 140 
 141 void InterpreterMacroAssembler::check_and_handle_earlyret(Register java_thread) {
 142   if (JvmtiExport::can_force_early_return()) {
 143     Label L;
 144     ldr(rscratch1, Address(rthread, JavaThread::jvmti_thread_state_offset()));
 145     cbz(rscratch1, L); // if (thread->jvmti_thread_state() == nullptr) exit;
 146 
 147     // Initiate earlyret handling only if it is not already being processed.
 148     // If the flag has the earlyret_processing bit set, it means that this code
 149     // is called *during* earlyret handling - we don't want to reenter.
 150     ldrw(rscratch1, Address(rscratch1, JvmtiThreadState::earlyret_state_offset()));
 151     cmpw(rscratch1, JvmtiThreadState::earlyret_pending);
 152     br(Assembler::NE, L);
 153 
 154     // Call Interpreter::remove_activation_early_entry() to get the address of the
 155     // same-named entrypoint in the generated interpreter code.
 156     ldr(rscratch1, Address(rthread, JavaThread::jvmti_thread_state_offset()));
 157     ldrw(rscratch1, Address(rscratch1, JvmtiThreadState::earlyret_tos_offset()));
 158     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), rscratch1);
 159     br(r0);
 160     bind(L);
 161   }
 162 }
 163 
 164 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(
 165   Register reg,
 166   int bcp_offset) {
 167   assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");
 168   ldrh(reg, Address(rbcp, bcp_offset));
 169   rev16(reg, reg);
 170 }
 171 
 172 void InterpreterMacroAssembler::get_dispatch() {
 173   uint64_t offset;
 174   adrp(rdispatch, ExternalAddress((address)Interpreter::dispatch_table()), offset);
 175   // Use add() here after ARDP, rather than lea().
 176   // lea() does not generate anything if its offset is zero.
 177   // However, relocs expect to find either an ADD or a load/store
 178   // insn after an ADRP.  add() always generates an ADD insn, even
 179   // for add(Rn, Rn, 0).
 180   add(rdispatch, rdispatch, offset);
 181 }
 182 
 183 void InterpreterMacroAssembler::get_cache_index_at_bcp(Register index,
 184                                                        int bcp_offset,
 185                                                        size_t index_size) {
 186   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 187   if (index_size == sizeof(u2)) {
 188     load_unsigned_short(index, Address(rbcp, bcp_offset));
 189   } else if (index_size == sizeof(u4)) {
 190     // assert(EnableInvokeDynamic, "giant index used only for JSR 292");
 191     ldrw(index, Address(rbcp, bcp_offset));
 192   } else if (index_size == sizeof(u1)) {
 193     load_unsigned_byte(index, Address(rbcp, bcp_offset));
 194   } else {
 195     ShouldNotReachHere();
 196   }
 197 }
 198 
 199 void InterpreterMacroAssembler::get_method_counters(Register method,
 200                                                     Register mcs, Label& skip) {
 201   Label has_counters;
 202   ldr(mcs, Address(method, Method::method_counters_offset()));
 203   cbnz(mcs, has_counters);
 204   call_VM(noreg, CAST_FROM_FN_PTR(address,
 205           InterpreterRuntime::build_method_counters), method);
 206   ldr(mcs, Address(method, Method::method_counters_offset()));
 207   cbz(mcs, skip); // No MethodCounters allocated, OutOfMemory
 208   bind(has_counters);
 209 }
 210 
 211 // Load object from cpool->resolved_references(index)
 212 void InterpreterMacroAssembler::load_resolved_reference_at_index(
 213                                            Register result, Register index, Register tmp) {
 214   assert_different_registers(result, index);
 215 
 216   get_constant_pool(result);
 217   // load pointer for resolved_references[] objArray
 218   ldr(result, Address(result, ConstantPool::cache_offset()));
 219   ldr(result, Address(result, ConstantPoolCache::resolved_references_offset()));
 220   resolve_oop_handle(result, tmp, rscratch2);
 221   // Add in the index
 222   add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 223   load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)), tmp, rscratch2);
 224 }
 225 
 226 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
 227                              Register cpool, Register index, Register klass, Register temp) {
 228   add(temp, cpool, index, LSL, LogBytesPerWord);
 229   ldrh(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index
 230   ldr(klass, Address(cpool,  ConstantPool::resolved_klasses_offset())); // klass = cpool->_resolved_klasses
 231   add(klass, klass, temp, LSL, LogBytesPerWord);
 232   ldr(klass, Address(klass, Array<Klass*>::base_offset_in_bytes()));
 233 }
 234 
 235 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
 236 // subtype of super_klass.
 237 //
 238 // Args:
 239 //      r0: superklass
 240 //      Rsub_klass: subklass
 241 //
 242 // Kills:
 243 //      r2, r5
 244 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
 245                                                   Label& ok_is_subtype) {
 246   assert(Rsub_klass != r0, "r0 holds superklass");
 247   assert(Rsub_klass != r2, "r2 holds 2ndary super array length");
 248   assert(Rsub_klass != r5, "r5 holds 2ndary super array scan ptr");
 249 
 250   // Profile the not-null value's klass.
 251   profile_typecheck(r2, Rsub_klass, r5); // blows r2, reloads r5
 252 
 253   // Do the check.
 254   check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2
 255 }
 256 
 257 // Java Expression Stack
 258 
 259 void InterpreterMacroAssembler::pop_ptr(Register r) {
 260   ldr(r, post(esp, wordSize));
 261 }
 262 
 263 void InterpreterMacroAssembler::pop_i(Register r) {
 264   ldrw(r, post(esp, wordSize));
 265 }
 266 
 267 void InterpreterMacroAssembler::pop_l(Register r) {
 268   ldr(r, post(esp, 2 * Interpreter::stackElementSize));
 269 }
 270 
 271 void InterpreterMacroAssembler::push_ptr(Register r) {
 272   str(r, pre(esp, -wordSize));
 273  }
 274 
 275 void InterpreterMacroAssembler::push_i(Register r) {
 276   str(r, pre(esp, -wordSize));
 277 }
 278 
 279 void InterpreterMacroAssembler::push_l(Register r) {
 280   str(zr, pre(esp, -wordSize));
 281   str(r, pre(esp, - wordSize));
 282 }
 283 
 284 void InterpreterMacroAssembler::pop_f(FloatRegister r) {
 285   ldrs(r, post(esp, wordSize));
 286 }
 287 
 288 void InterpreterMacroAssembler::pop_d(FloatRegister r) {
 289   ldrd(r, post(esp, 2 * Interpreter::stackElementSize));
 290 }
 291 
 292 void InterpreterMacroAssembler::push_f(FloatRegister r) {
 293   strs(r, pre(esp, -wordSize));
 294 }
 295 
 296 void InterpreterMacroAssembler::push_d(FloatRegister r) {
 297   strd(r, pre(esp, 2* -wordSize));
 298 }
 299 
 300 void InterpreterMacroAssembler::pop(TosState state) {
 301   switch (state) {
 302   case atos: pop_ptr();                 break;
 303   case btos:
 304   case ztos:
 305   case ctos:
 306   case stos:
 307   case itos: pop_i();                   break;
 308   case ltos: pop_l();                   break;
 309   case ftos: pop_f();                   break;
 310   case dtos: pop_d();                   break;
 311   case vtos: /* nothing to do */        break;
 312   default:   ShouldNotReachHere();
 313   }
 314   interp_verify_oop(r0, state);
 315 }
 316 
 317 void InterpreterMacroAssembler::push(TosState state) {
 318   interp_verify_oop(r0, state);
 319   switch (state) {
 320   case atos: push_ptr();                break;
 321   case btos:
 322   case ztos:
 323   case ctos:
 324   case stos:
 325   case itos: push_i();                  break;
 326   case ltos: push_l();                  break;
 327   case ftos: push_f();                  break;
 328   case dtos: push_d();                  break;
 329   case vtos: /* nothing to do */        break;
 330   default  : ShouldNotReachHere();
 331   }
 332 }
 333 
 334 // Helpers for swap and dup
 335 void InterpreterMacroAssembler::load_ptr(int n, Register val) {
 336   ldr(val, Address(esp, Interpreter::expr_offset_in_bytes(n)));
 337 }
 338 
 339 void InterpreterMacroAssembler::store_ptr(int n, Register val) {
 340   str(val, Address(esp, Interpreter::expr_offset_in_bytes(n)));
 341 }
 342 
 343 void InterpreterMacroAssembler::load_float(Address src) {
 344   ldrs(v0, src);
 345 }
 346 
 347 void InterpreterMacroAssembler::load_double(Address src) {
 348   ldrd(v0, src);
 349 }
 350 
 351 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() {
 352   // set sender sp
 353   mov(r19_sender_sp, sp);
 354   // record last_sp
 355   sub(rscratch1, esp, rfp);
 356   asr(rscratch1, rscratch1, Interpreter::logStackElementSize);
 357   str(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
 358 }
 359 
 360 // Jump to from_interpreted entry of a call unless single stepping is possible
 361 // in this thread in which case we must call the i2i entry
 362 void InterpreterMacroAssembler::jump_from_interpreted(Register method, Register temp) {
 363   prepare_to_jump_from_interpreted();
 364 
 365   if (JvmtiExport::can_post_interpreter_events()) {
 366     Label run_compiled_code;
 367     // JVMTI events, such as single-stepping, are implemented partly by avoiding running
 368     // compiled code in threads for which the event is enabled.  Check here for
 369     // interp_only_mode if these events CAN be enabled.
 370     ldrw(rscratch1, Address(rthread, JavaThread::interp_only_mode_offset()));
 371     cbzw(rscratch1, run_compiled_code);
 372     ldr(rscratch1, Address(method, Method::interpreter_entry_offset()));
 373     br(rscratch1);
 374     bind(run_compiled_code);
 375   }
 376 
 377   ldr(rscratch1, Address(method, Method::from_interpreted_offset()));
 378   br(rscratch1);
 379 }
 380 
 381 // The following two routines provide a hook so that an implementation
 382 // can schedule the dispatch in two parts.  amd64 does not do this.
 383 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) {
 384 }
 385 
 386 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {
 387     dispatch_next(state, step);
 388 }
 389 
 390 void InterpreterMacroAssembler::dispatch_base(TosState state,
 391                                               address* table,
 392                                               bool verifyoop,
 393                                               bool generate_poll) {
 394   if (VerifyActivationFrameSize) {
 395     Label L;
 396     sub(rscratch2, rfp, esp);
 397     int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize;
 398     subs(rscratch2, rscratch2, min_frame_size);
 399     br(Assembler::GE, L);
 400     stop("broken stack frame");
 401     bind(L);
 402   }
 403   if (verifyoop) {
 404     interp_verify_oop(r0, state);
 405   }
 406 
 407   Label safepoint;
 408   address* const safepoint_table = Interpreter::safept_table(state);
 409   bool needs_thread_local_poll = generate_poll && table != safepoint_table;
 410 
 411   if (needs_thread_local_poll) {
 412     NOT_PRODUCT(block_comment("Thread-local Safepoint poll"));
 413     ldr(rscratch2, Address(rthread, JavaThread::polling_word_offset()));
 414     tbnz(rscratch2, exact_log2(SafepointMechanism::poll_bit()), safepoint);
 415   }
 416 
 417   if (table == Interpreter::dispatch_table(state)) {
 418     addw(rscratch2, rscratch1, Interpreter::distance_from_dispatch_table(state));
 419     ldr(rscratch2, Address(rdispatch, rscratch2, Address::uxtw(3)));
 420   } else {
 421     mov(rscratch2, (address)table);
 422     ldr(rscratch2, Address(rscratch2, rscratch1, Address::uxtw(3)));
 423   }
 424   br(rscratch2);
 425 
 426   if (needs_thread_local_poll) {
 427     bind(safepoint);
 428     lea(rscratch2, ExternalAddress((address)safepoint_table));
 429     ldr(rscratch2, Address(rscratch2, rscratch1, Address::uxtw(3)));
 430     br(rscratch2);
 431   }
 432 }
 433 
 434 void InterpreterMacroAssembler::dispatch_only(TosState state, bool generate_poll) {
 435   dispatch_base(state, Interpreter::dispatch_table(state), true, generate_poll);
 436 }
 437 
 438 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) {
 439   dispatch_base(state, Interpreter::normal_table(state));
 440 }
 441 
 442 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
 443   dispatch_base(state, Interpreter::normal_table(state), false);
 444 }
 445 
 446 
 447 void InterpreterMacroAssembler::dispatch_next(TosState state, int step, bool generate_poll) {
 448   // load next bytecode
 449   ldrb(rscratch1, Address(pre(rbcp, step)));
 450   dispatch_base(state, Interpreter::dispatch_table(state), /*verifyoop*/true, generate_poll);
 451 }
 452 
 453 void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) {
 454   // load current bytecode
 455   ldrb(rscratch1, Address(rbcp, 0));
 456   dispatch_base(state, table);
 457 }
 458 
 459 // remove activation
 460 //
 461 // Apply stack watermark barrier.
 462 // Unlock the receiver if this is a synchronized method.
 463 // Unlock any Java monitors from synchronized blocks.
 464 // Remove the activation from the stack.
 465 //
 466 // If there are locked Java monitors
 467 //    If throw_monitor_exception
 468 //       throws IllegalMonitorStateException
 469 //    Else if install_monitor_exception
 470 //       installs IllegalMonitorStateException
 471 //    Else
 472 //       no error processing
 473 void InterpreterMacroAssembler::remove_activation(
 474         TosState state,
 475         bool throw_monitor_exception,
 476         bool install_monitor_exception,
 477         bool notify_jvmdi) {
 478   // Note: Registers r3 xmm0 may be in use for the
 479   // result check if synchronized method
 480   Label unlocked, unlock, no_unlock;
 481 
 482   // The below poll is for the stack watermark barrier. It allows fixing up frames lazily,
 483   // that would normally not be safe to use. Such bad returns into unsafe territory of
 484   // the stack, will call InterpreterRuntime::at_unwind.
 485   Label slow_path;
 486   Label fast_path;
 487   safepoint_poll(slow_path, true /* at_return */, false /* acquire */, false /* in_nmethod */);
 488   br(Assembler::AL, fast_path);
 489   bind(slow_path);
 490   push(state);
 491   set_last_Java_frame(esp, rfp, (address)pc(), rscratch1);
 492   super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
 493   reset_last_Java_frame(true);
 494   pop(state);
 495   bind(fast_path);
 496 
 497   // get the value of _do_not_unlock_if_synchronized into r3
 498   const Address do_not_unlock_if_synchronized(rthread,
 499     in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
 500   ldrb(r3, do_not_unlock_if_synchronized);
 501   strb(zr, do_not_unlock_if_synchronized); // reset the flag
 502 
 503  // get method access flags
 504   ldr(r1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
 505   ldrh(r2, Address(r1, Method::access_flags_offset()));
 506   tbz(r2, exact_log2(JVM_ACC_SYNCHRONIZED), unlocked);
 507 
 508   // Don't unlock anything if the _do_not_unlock_if_synchronized flag
 509   // is set.
 510   cbnz(r3, no_unlock);
 511 
 512   // unlock monitor
 513   push(state); // save result
 514 
 515   // BasicObjectLock will be first in list, since this is a
 516   // synchronized method. However, need to check that the object has
 517   // not been unlocked by an explicit monitorexit bytecode.
 518   const Address monitor(rfp, frame::interpreter_frame_initial_sp_offset *
 519                         wordSize - (int) sizeof(BasicObjectLock));
 520   // We use c_rarg1 so that if we go slow path it will be the correct
 521   // register for unlock_object to pass to VM directly
 522   lea(c_rarg1, monitor); // address of first monitor
 523 
 524   ldr(r0, Address(c_rarg1, BasicObjectLock::obj_offset()));
 525   cbnz(r0, unlock);
 526 
 527   pop(state);
 528   if (throw_monitor_exception) {
 529     // Entry already unlocked, need to throw exception
 530     call_VM(noreg, CAST_FROM_FN_PTR(address,
 531                    InterpreterRuntime::throw_illegal_monitor_state_exception));
 532     should_not_reach_here();
 533   } else {
 534     // Monitor already unlocked during a stack unroll. If requested,
 535     // install an illegal_monitor_state_exception.  Continue with
 536     // stack unrolling.
 537     if (install_monitor_exception) {
 538       call_VM(noreg, CAST_FROM_FN_PTR(address,
 539                      InterpreterRuntime::new_illegal_monitor_state_exception));
 540     }
 541     b(unlocked);
 542   }
 543 
 544   bind(unlock);
 545   unlock_object(c_rarg1);
 546   pop(state);
 547 
 548   // Check that for block-structured locking (i.e., that all locked
 549   // objects has been unlocked)
 550   bind(unlocked);
 551 
 552   // r0: Might contain return value
 553 
 554   // Check that all monitors are unlocked
 555   {
 556     Label loop, exception, entry, restart;
 557     const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
 558     const Address monitor_block_top(
 559         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
 560     const Address monitor_block_bot(
 561         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
 562 
 563     bind(restart);
 564     // We use c_rarg1 so that if we go slow path it will be the correct
 565     // register for unlock_object to pass to VM directly
 566     ldr(c_rarg1, monitor_block_top); // derelativize pointer
 567     lea(c_rarg1, Address(rfp, c_rarg1, Address::lsl(Interpreter::logStackElementSize)));
 568     // c_rarg1 points to current entry, starting with top-most entry
 569 
 570     lea(r19, monitor_block_bot);  // points to word before bottom of
 571                                   // monitor block
 572     b(entry);
 573 
 574     // Entry already locked, need to throw exception
 575     bind(exception);
 576 
 577     if (throw_monitor_exception) {
 578       // Throw exception
 579       MacroAssembler::call_VM(noreg,
 580                               CAST_FROM_FN_PTR(address, InterpreterRuntime::
 581                                    throw_illegal_monitor_state_exception));
 582       should_not_reach_here();
 583     } else {
 584       // Stack unrolling. Unlock object and install illegal_monitor_exception.
 585       // Unlock does not block, so don't have to worry about the frame.
 586       // We don't have to preserve c_rarg1 since we are going to throw an exception.
 587 
 588       push(state);
 589       unlock_object(c_rarg1);
 590       pop(state);
 591 
 592       if (install_monitor_exception) {
 593         call_VM(noreg, CAST_FROM_FN_PTR(address,
 594                                         InterpreterRuntime::
 595                                         new_illegal_monitor_state_exception));
 596       }
 597 
 598       b(restart);
 599     }
 600 
 601     bind(loop);
 602     // check if current entry is used
 603     ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset()));
 604     cbnz(rscratch1, exception);
 605 
 606     add(c_rarg1, c_rarg1, entry_size); // otherwise advance to next entry
 607     bind(entry);
 608     cmp(c_rarg1, r19); // check if bottom reached
 609     br(Assembler::NE, loop); // if not at bottom then check this entry
 610   }
 611 
 612   bind(no_unlock);
 613 
 614   // jvmti support
 615   if (notify_jvmdi) {
 616     notify_method_exit(state, NotifyJVMTI);    // preserve TOSCA
 617   } else {
 618     notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
 619   }
 620 
 621   // remove activation
 622   // get sender esp
 623   ldr(rscratch2,
 624       Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
 625   if (StackReservedPages > 0) {
 626     // testing if reserved zone needs to be re-enabled
 627     Label no_reserved_zone_enabling;
 628 
 629     // check if already enabled - if so no re-enabling needed
 630     assert(sizeof(StackOverflow::StackGuardState) == 4, "unexpected size");
 631     ldrw(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
 632     cmpw(rscratch1, (u1)StackOverflow::stack_guard_enabled);
 633     br(Assembler::EQ, no_reserved_zone_enabling);
 634 
 635     // look for an overflow into the stack reserved zone, i.e.
 636     // interpreter_frame_sender_sp <= JavaThread::reserved_stack_activation
 637     ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
 638     cmp(rscratch2, rscratch1);
 639     br(Assembler::LS, no_reserved_zone_enabling);
 640 
 641     call_VM_leaf(
 642       CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
 643     call_VM(noreg, CAST_FROM_FN_PTR(address,
 644                    InterpreterRuntime::throw_delayed_StackOverflowError));
 645     should_not_reach_here();
 646 
 647     bind(no_reserved_zone_enabling);
 648   }
 649 
 650   // restore sender esp
 651   mov(esp, rscratch2);
 652   // remove frame anchor
 653   leave();
 654   // If we're returning to interpreted code we will shortly be
 655   // adjusting SP to allow some space for ESP.  If we're returning to
 656   // compiled code the saved sender SP was saved in sender_sp, so this
 657   // restores it.
 658   andr(sp, esp, -16);
 659 }
 660 
 661 // Lock object
 662 //
 663 // Args:
 664 //      c_rarg1: BasicObjectLock to be used for locking
 665 //
 666 // Kills:
 667 //      r0
 668 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, .. (param regs)
 669 //      rscratch1, rscratch2 (scratch regs)
 670 void InterpreterMacroAssembler::lock_object(Register lock_reg)
 671 {
 672   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
 673   if (LockingMode == LM_MONITOR) {
 674     call_VM_preemptable(noreg,
 675             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
 676             lock_reg);
 677   } else {
 678     Label count, done;
 679 
 680     const Register swap_reg = r0;
 681     const Register tmp = c_rarg2;
 682     const Register obj_reg = c_rarg3; // Will contain the oop
 683     const Register tmp2 = c_rarg4;
 684     const Register tmp3 = c_rarg5;
 685 
 686     const int obj_offset = in_bytes(BasicObjectLock::obj_offset());
 687     const int lock_offset = in_bytes(BasicObjectLock::lock_offset());
 688     const int mark_offset = lock_offset +
 689                             BasicLock::displaced_header_offset_in_bytes();
 690 
 691     Label slow_case;
 692 
 693     // Load object pointer into obj_reg %c_rarg3
 694     ldr(obj_reg, Address(lock_reg, obj_offset));
 695 
 696     if (DiagnoseSyncOnValueBasedClasses != 0) {
 697       load_klass(tmp, obj_reg);
 698       ldrb(tmp, Address(tmp, Klass::misc_flags_offset()));
 699       tst(tmp, KlassFlags::_misc_is_value_based_class);
 700       br(Assembler::NE, slow_case);
 701     }
 702 
 703     if (LockingMode == LM_LIGHTWEIGHT) {
 704       lightweight_lock(lock_reg, obj_reg, tmp, tmp2, tmp3, slow_case);
 705       b(done);
 706     } else if (LockingMode == LM_LEGACY) {
 707       // Load (object->mark() | 1) into swap_reg
 708       ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
 709       orr(swap_reg, rscratch1, 1);
 710 
 711       // Save (object->mark() | 1) into BasicLock's displaced header
 712       str(swap_reg, Address(lock_reg, mark_offset));
 713 
 714       assert(lock_offset == 0,
 715              "displached header must be first word in BasicObjectLock");
 716 
 717       Label fail;
 718       cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
 719 
 720       // Fast check for recursive lock.
 721       //
 722       // Can apply the optimization only if this is a stack lock
 723       // allocated in this thread. For efficiency, we can focus on
 724       // recently allocated stack locks (instead of reading the stack
 725       // base and checking whether 'mark' points inside the current
 726       // thread stack):
 727       //  1) (mark & 7) == 0, and
 728       //  2) sp <= mark < mark + os::pagesize()
 729       //
 730       // Warning: sp + os::pagesize can overflow the stack base. We must
 731       // neither apply the optimization for an inflated lock allocated
 732       // just above the thread stack (this is why condition 1 matters)
 733       // nor apply the optimization if the stack lock is inside the stack
 734       // of another thread. The latter is avoided even in case of overflow
 735       // because we have guard pages at the end of all stacks. Hence, if
 736       // we go over the stack base and hit the stack of another thread,
 737       // this should not be in a writeable area that could contain a
 738       // stack lock allocated by that thread. As a consequence, a stack
 739       // lock less than page size away from sp is guaranteed to be
 740       // owned by the current thread.
 741       //
 742       // These 3 tests can be done by evaluating the following
 743       // expression: ((mark - sp) & (7 - os::vm_page_size())),
 744       // assuming both stack pointer and pagesize have their
 745       // least significant 3 bits clear.
 746       // NOTE: the mark is in swap_reg %r0 as the result of cmpxchg
 747       // NOTE2: aarch64 does not like to subtract sp from rn so take a
 748       // copy
 749       mov(rscratch1, sp);
 750       sub(swap_reg, swap_reg, rscratch1);
 751       ands(swap_reg, swap_reg, (uint64_t)(7 - (int)os::vm_page_size()));
 752 
 753       // Save the test result, for recursive case, the result is zero
 754       str(swap_reg, Address(lock_reg, mark_offset));
 755       br(Assembler::NE, slow_case);
 756 
 757       bind(count);
 758       inc_held_monitor_count(rscratch1);
 759       b(done);
 760     }
 761     bind(slow_case);
 762 
 763     // Call the runtime routine for slow case
 764     call_VM_preemptable(noreg,
 765             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
 766             lock_reg);
 767 
 768     bind(done);
 769   }
 770 }
 771 
 772 
 773 // Unlocks an object. Used in monitorexit bytecode and
 774 // remove_activation.  Throws an IllegalMonitorException if object is
 775 // not locked by current thread.
 776 //
 777 // Args:
 778 //      c_rarg1: BasicObjectLock for lock
 779 //
 780 // Kills:
 781 //      r0
 782 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
 783 //      rscratch1, rscratch2 (scratch regs)
 784 void InterpreterMacroAssembler::unlock_object(Register lock_reg)
 785 {
 786   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
 787 
 788   if (LockingMode == LM_MONITOR) {
 789     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
 790   } else {
 791     Label count, done;
 792 
 793     const Register swap_reg   = r0;
 794     const Register header_reg = c_rarg2;  // Will contain the old oopMark
 795     const Register obj_reg    = c_rarg3;  // Will contain the oop
 796     const Register tmp_reg    = c_rarg4;  // Temporary used by lightweight_unlock
 797 
 798     save_bcp(); // Save in case of exception
 799 
 800     if (LockingMode != LM_LIGHTWEIGHT) {
 801       // Convert from BasicObjectLock structure to object and BasicLock
 802       // structure Store the BasicLock address into %r0
 803       lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset()));
 804     }
 805 
 806     // Load oop into obj_reg(%c_rarg3)
 807     ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset()));
 808 
 809     // Free entry
 810     str(zr, Address(lock_reg, BasicObjectLock::obj_offset()));
 811 
 812     Label slow_case;
 813     if (LockingMode == LM_LIGHTWEIGHT) {
 814       lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
 815       b(done);
 816     } else if (LockingMode == LM_LEGACY) {
 817       // Load the old header from BasicLock structure
 818       ldr(header_reg, Address(swap_reg,
 819                               BasicLock::displaced_header_offset_in_bytes()));
 820 
 821       // Test for recursion
 822       cbz(header_reg, count);
 823 
 824       // Atomic swap back the old header
 825       cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, count, &slow_case);
 826 
 827       bind(count);
 828       dec_held_monitor_count(rscratch1);
 829       b(done);
 830     }
 831 
 832     bind(slow_case);
 833     // Call the runtime routine for slow case.
 834     str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset())); // restore obj
 835     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
 836     bind(done);
 837     restore_bcp();
 838   }
 839 }
 840 
 841 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
 842                                                          Label& zero_continue) {
 843   assert(ProfileInterpreter, "must be profiling interpreter");
 844   ldr(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
 845   cbz(mdp, zero_continue);
 846 }
 847 
 848 // Set the method data pointer for the current bcp.
 849 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
 850   assert(ProfileInterpreter, "must be profiling interpreter");
 851   Label set_mdp;
 852   stp(r0, r1, Address(pre(sp, -2 * wordSize)));
 853 
 854   // Test MDO to avoid the call if it is null.
 855   ldr(r0, Address(rmethod, in_bytes(Method::method_data_offset())));
 856   cbz(r0, set_mdp);
 857   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rmethod, rbcp);
 858   // r0: mdi
 859   // mdo is guaranteed to be non-zero here, we checked for it before the call.
 860   ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
 861   lea(r1, Address(r1, in_bytes(MethodData::data_offset())));
 862   add(r0, r1, r0);
 863   str(r0, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
 864   bind(set_mdp);
 865   ldp(r0, r1, Address(post(sp, 2 * wordSize)));
 866 }
 867 
 868 void InterpreterMacroAssembler::verify_method_data_pointer() {
 869   assert(ProfileInterpreter, "must be profiling interpreter");
 870 #ifdef ASSERT
 871   Label verify_continue;
 872   stp(r0, r1, Address(pre(sp, -2 * wordSize)));
 873   stp(r2, r3, Address(pre(sp, -2 * wordSize)));
 874   test_method_data_pointer(r3, verify_continue); // If mdp is zero, continue
 875   get_method(r1);
 876 
 877   // If the mdp is valid, it will point to a DataLayout header which is
 878   // consistent with the bcp.  The converse is highly probable also.
 879   ldrsh(r2, Address(r3, in_bytes(DataLayout::bci_offset())));
 880   ldr(rscratch1, Address(r1, Method::const_offset()));
 881   add(r2, r2, rscratch1, Assembler::LSL);
 882   lea(r2, Address(r2, ConstMethod::codes_offset()));
 883   cmp(r2, rbcp);
 884   br(Assembler::EQ, verify_continue);
 885   // r1: method
 886   // rbcp: bcp // rbcp == 22
 887   // r3: mdp
 888   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp),
 889                r1, rbcp, r3);
 890   bind(verify_continue);
 891   ldp(r2, r3, Address(post(sp, 2 * wordSize)));
 892   ldp(r0, r1, Address(post(sp, 2 * wordSize)));
 893 #endif // ASSERT
 894 }
 895 
 896 
 897 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in,
 898                                                 int constant,
 899                                                 Register value) {
 900   assert(ProfileInterpreter, "must be profiling interpreter");
 901   Address data(mdp_in, constant);
 902   str(value, data);
 903 }
 904 
 905 
 906 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
 907                                                       int constant,
 908                                                       bool decrement) {
 909   increment_mdp_data_at(mdp_in, noreg, constant, decrement);
 910 }
 911 
 912 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
 913                                                       Register reg,
 914                                                       int constant,
 915                                                       bool decrement) {
 916   assert(ProfileInterpreter, "must be profiling interpreter");
 917   // %%% this does 64bit counters at best it is wasting space
 918   // at worst it is a rare bug when counters overflow
 919 
 920   assert_different_registers(rscratch2, rscratch1, mdp_in, reg);
 921 
 922   Address addr1(mdp_in, constant);
 923   Address addr2(rscratch2, reg, Address::lsl(0));
 924   Address &addr = addr1;
 925   if (reg != noreg) {
 926     lea(rscratch2, addr1);
 927     addr = addr2;
 928   }
 929 
 930   if (decrement) {
 931     // Decrement the register.  Set condition codes.
 932     // Intel does this
 933     // addptr(data, (int32_t) -DataLayout::counter_increment);
 934     // If the decrement causes the counter to overflow, stay negative
 935     // Label L;
 936     // jcc(Assembler::negative, L);
 937     // addptr(data, (int32_t) DataLayout::counter_increment);
 938     // so we do this
 939     ldr(rscratch1, addr);
 940     subs(rscratch1, rscratch1, (unsigned)DataLayout::counter_increment);
 941     Label L;
 942     br(Assembler::LO, L);       // skip store if counter underflow
 943     str(rscratch1, addr);
 944     bind(L);
 945   } else {
 946     assert(DataLayout::counter_increment == 1,
 947            "flow-free idiom only works with 1");
 948     // Intel does this
 949     // Increment the register.  Set carry flag.
 950     // addptr(data, DataLayout::counter_increment);
 951     // If the increment causes the counter to overflow, pull back by 1.
 952     // sbbptr(data, (int32_t)0);
 953     // so we do this
 954     ldr(rscratch1, addr);
 955     adds(rscratch1, rscratch1, DataLayout::counter_increment);
 956     Label L;
 957     br(Assembler::CS, L);       // skip store if counter overflow
 958     str(rscratch1, addr);
 959     bind(L);
 960   }
 961 }
 962 
 963 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in,
 964                                                 int flag_byte_constant) {
 965   assert(ProfileInterpreter, "must be profiling interpreter");
 966   int flags_offset = in_bytes(DataLayout::flags_offset());
 967   // Set the flag
 968   ldrb(rscratch1, Address(mdp_in, flags_offset));
 969   orr(rscratch1, rscratch1, flag_byte_constant);
 970   strb(rscratch1, Address(mdp_in, flags_offset));
 971 }
 972 
 973 
 974 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in,
 975                                                  int offset,
 976                                                  Register value,
 977                                                  Register test_value_out,
 978                                                  Label& not_equal_continue) {
 979   assert(ProfileInterpreter, "must be profiling interpreter");
 980   if (test_value_out == noreg) {
 981     ldr(rscratch1, Address(mdp_in, offset));
 982     cmp(value, rscratch1);
 983   } else {
 984     // Put the test value into a register, so caller can use it:
 985     ldr(test_value_out, Address(mdp_in, offset));
 986     cmp(value, test_value_out);
 987   }
 988   br(Assembler::NE, not_equal_continue);
 989 }
 990 
 991 
 992 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in,
 993                                                      int offset_of_disp) {
 994   assert(ProfileInterpreter, "must be profiling interpreter");
 995   ldr(rscratch1, Address(mdp_in, offset_of_disp));
 996   add(mdp_in, mdp_in, rscratch1, LSL);
 997   str(mdp_in, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
 998 }
 999 
1000 
1001 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in,
1002                                                      Register reg,
1003                                                      int offset_of_disp) {
1004   assert(ProfileInterpreter, "must be profiling interpreter");
1005   lea(rscratch1, Address(mdp_in, offset_of_disp));
1006   ldr(rscratch1, Address(rscratch1, reg, Address::lsl(0)));
1007   add(mdp_in, mdp_in, rscratch1, LSL);
1008   str(mdp_in, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1009 }
1010 
1011 
1012 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in,
1013                                                        int constant) {
1014   assert(ProfileInterpreter, "must be profiling interpreter");
1015   add(mdp_in, mdp_in, (unsigned)constant);
1016   str(mdp_in, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1017 }
1018 
1019 
1020 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {
1021   assert(ProfileInterpreter, "must be profiling interpreter");
1022   // save/restore across call_VM
1023   stp(zr, return_bci, Address(pre(sp, -2 * wordSize)));
1024   call_VM(noreg,
1025           CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret),
1026           return_bci);
1027   ldp(zr, return_bci, Address(post(sp, 2 * wordSize)));
1028 }
1029 
1030 
1031 void InterpreterMacroAssembler::profile_taken_branch(Register mdp,
1032                                                      Register bumped_count) {
1033   if (ProfileInterpreter) {
1034     Label profile_continue;
1035 
1036     // If no method data exists, go to profile_continue.
1037     // Otherwise, assign to mdp
1038     test_method_data_pointer(mdp, profile_continue);
1039 
1040     // We are taking a branch.  Increment the taken count.
1041     // We inline increment_mdp_data_at to return bumped_count in a register
1042     //increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
1043     Address data(mdp, in_bytes(JumpData::taken_offset()));
1044     ldr(bumped_count, data);
1045     assert(DataLayout::counter_increment == 1,
1046             "flow-free idiom only works with 1");
1047     // Intel does this to catch overflow
1048     // addptr(bumped_count, DataLayout::counter_increment);
1049     // sbbptr(bumped_count, 0);
1050     // so we do this
1051     adds(bumped_count, bumped_count, DataLayout::counter_increment);
1052     Label L;
1053     br(Assembler::CS, L);       // skip store if counter overflow
1054     str(bumped_count, data);
1055     bind(L);
1056     // The method data pointer needs to be updated to reflect the new target.
1057     update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
1058     bind(profile_continue);
1059   }
1060 }
1061 
1062 
1063 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
1064   if (ProfileInterpreter) {
1065     Label profile_continue;
1066 
1067     // If no method data exists, go to profile_continue.
1068     test_method_data_pointer(mdp, profile_continue);
1069 
1070     // We are taking a branch.  Increment the not taken count.
1071     increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
1072 
1073     // The method data pointer needs to be updated to correspond to
1074     // the next bytecode
1075     update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
1076     bind(profile_continue);
1077   }
1078 }
1079 
1080 
1081 void InterpreterMacroAssembler::profile_call(Register mdp) {
1082   if (ProfileInterpreter) {
1083     Label profile_continue;
1084 
1085     // If no method data exists, go to profile_continue.
1086     test_method_data_pointer(mdp, profile_continue);
1087 
1088     // We are making a call.  Increment the count.
1089     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1090 
1091     // The method data pointer needs to be updated to reflect the new target.
1092     update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1093     bind(profile_continue);
1094   }
1095 }
1096 
1097 void InterpreterMacroAssembler::profile_final_call(Register mdp) {
1098   if (ProfileInterpreter) {
1099     Label profile_continue;
1100 
1101     // If no method data exists, go to profile_continue.
1102     test_method_data_pointer(mdp, profile_continue);
1103 
1104     // We are making a call.  Increment the count.
1105     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1106 
1107     // The method data pointer needs to be updated to reflect the new target.
1108     update_mdp_by_constant(mdp,
1109                            in_bytes(VirtualCallData::
1110                                     virtual_call_data_size()));
1111     bind(profile_continue);
1112   }
1113 }
1114 
1115 
1116 void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
1117                                                      Register mdp,
1118                                                      Register reg2,
1119                                                      bool receiver_can_be_null) {
1120   if (ProfileInterpreter) {
1121     Label profile_continue;
1122 
1123     // If no method data exists, go to profile_continue.
1124     test_method_data_pointer(mdp, profile_continue);
1125 
1126     Label skip_receiver_profile;
1127     if (receiver_can_be_null) {
1128       Label not_null;
1129       // We are making a call.  Increment the count for null receiver.
1130       increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1131       b(skip_receiver_profile);
1132       bind(not_null);
1133     }
1134 
1135     // Record the receiver type.
1136     record_klass_in_profile(receiver, mdp, reg2);
1137     bind(skip_receiver_profile);
1138 
1139     // The method data pointer needs to be updated to reflect the new target.
1140     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1141     bind(profile_continue);
1142   }
1143 }
1144 
1145 // This routine creates a state machine for updating the multi-row
1146 // type profile at a virtual call site (or other type-sensitive bytecode).
1147 // The machine visits each row (of receiver/count) until the receiver type
1148 // is found, or until it runs out of rows.  At the same time, it remembers
1149 // the location of the first empty row.  (An empty row records null for its
1150 // receiver, and can be allocated for a newly-observed receiver type.)
1151 // Because there are two degrees of freedom in the state, a simple linear
1152 // search will not work; it must be a decision tree.  Hence this helper
1153 // function is recursive, to generate the required tree structured code.
1154 // It's the interpreter, so we are trading off code space for speed.
1155 // See below for example code.
1156 void InterpreterMacroAssembler::record_klass_in_profile_helper(
1157                                         Register receiver, Register mdp,
1158                                         Register reg2, int start_row,
1159                                         Label& done) {
1160   if (TypeProfileWidth == 0) {
1161     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1162   } else {
1163     record_item_in_profile_helper(receiver, mdp, reg2, 0, done, TypeProfileWidth,
1164         &VirtualCallData::receiver_offset, &VirtualCallData::receiver_count_offset);
1165   }
1166 }
1167 
1168 void InterpreterMacroAssembler::record_item_in_profile_helper(Register item, Register mdp,
1169                                         Register reg2, int start_row, Label& done, int total_rows,
1170                                         OffsetFunction item_offset_fn, OffsetFunction item_count_offset_fn) {
1171   int last_row = total_rows - 1;
1172   assert(start_row <= last_row, "must be work left to do");
1173   // Test this row for both the item and for null.
1174   // Take any of three different outcomes:
1175   //   1. found item => increment count and goto done
1176   //   2. found null => keep looking for case 1, maybe allocate this cell
1177   //   3. found something else => keep looking for cases 1 and 2
1178   // Case 3 is handled by a recursive call.
1179   for (int row = start_row; row <= last_row; row++) {
1180     Label next_test;
1181     bool test_for_null_also = (row == start_row);
1182 
1183     // See if the item is item[n].
1184     int item_offset = in_bytes(item_offset_fn(row));
1185     test_mdp_data_at(mdp, item_offset, item,
1186                      (test_for_null_also ? reg2 : noreg),
1187                      next_test);
1188     // (Reg2 now contains the item from the CallData.)
1189 
1190     // The item is item[n].  Increment count[n].
1191     int count_offset = in_bytes(item_count_offset_fn(row));
1192     increment_mdp_data_at(mdp, count_offset);
1193     b(done);
1194     bind(next_test);
1195 
1196     if (test_for_null_also) {
1197       Label found_null;
1198       // Failed the equality check on item[n]...  Test for null.
1199       if (start_row == last_row) {
1200         // The only thing left to do is handle the null case.
1201         cbz(reg2, found_null);
1202         // Item did not match any saved item and there is no empty row for it.
1203         // Increment total counter to indicate polymorphic case.
1204         increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1205         b(done);
1206         bind(found_null);
1207         break;
1208       }
1209       // Since null is rare, make it be the branch-taken case.
1210       cbz(reg2, found_null);
1211 
1212       // Put all the "Case 3" tests here.
1213       record_item_in_profile_helper(item, mdp, reg2, start_row + 1, done, total_rows,
1214         item_offset_fn, item_count_offset_fn);
1215 
1216       // Found a null.  Keep searching for a matching item,
1217       // but remember that this is an empty (unused) slot.
1218       bind(found_null);
1219     }
1220   }
1221 
1222   // In the fall-through case, we found no matching item, but we
1223   // observed the item[start_row] is null.
1224 
1225   // Fill in the item field and increment the count.
1226   int item_offset = in_bytes(item_offset_fn(start_row));
1227   set_mdp_data_at(mdp, item_offset, item);
1228   int count_offset = in_bytes(item_count_offset_fn(start_row));
1229   mov(reg2, DataLayout::counter_increment);
1230   set_mdp_data_at(mdp, count_offset, reg2);
1231   if (start_row > 0) {
1232     b(done);
1233   }
1234 }
1235 
1236 // Example state machine code for three profile rows:
1237 //   // main copy of decision tree, rooted at row[1]
1238 //   if (row[0].rec == rec) { row[0].incr(); goto done; }
1239 //   if (row[0].rec != nullptr) {
1240 //     // inner copy of decision tree, rooted at row[1]
1241 //     if (row[1].rec == rec) { row[1].incr(); goto done; }
1242 //     if (row[1].rec != nullptr) {
1243 //       // degenerate decision tree, rooted at row[2]
1244 //       if (row[2].rec == rec) { row[2].incr(); goto done; }
1245 //       if (row[2].rec != nullptr) { count.incr(); goto done; } // overflow
1246 //       row[2].init(rec); goto done;
1247 //     } else {
1248 //       // remember row[1] is empty
1249 //       if (row[2].rec == rec) { row[2].incr(); goto done; }
1250 //       row[1].init(rec); goto done;
1251 //     }
1252 //   } else {
1253 //     // remember row[0] is empty
1254 //     if (row[1].rec == rec) { row[1].incr(); goto done; }
1255 //     if (row[2].rec == rec) { row[2].incr(); goto done; }
1256 //     row[0].init(rec); goto done;
1257 //   }
1258 //   done:
1259 
1260 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
1261                                                         Register mdp, Register reg2) {
1262   assert(ProfileInterpreter, "must be profiling");
1263   Label done;
1264 
1265   record_klass_in_profile_helper(receiver, mdp, reg2, 0, done);
1266 
1267   bind (done);
1268 }
1269 
1270 void InterpreterMacroAssembler::profile_ret(Register return_bci,
1271                                             Register mdp) {
1272   if (ProfileInterpreter) {
1273     Label profile_continue;
1274     uint row;
1275 
1276     // If no method data exists, go to profile_continue.
1277     test_method_data_pointer(mdp, profile_continue);
1278 
1279     // Update the total ret count.
1280     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1281 
1282     for (row = 0; row < RetData::row_limit(); row++) {
1283       Label next_test;
1284 
1285       // See if return_bci is equal to bci[n]:
1286       test_mdp_data_at(mdp,
1287                        in_bytes(RetData::bci_offset(row)),
1288                        return_bci, noreg,
1289                        next_test);
1290 
1291       // return_bci is equal to bci[n].  Increment the count.
1292       increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)));
1293 
1294       // The method data pointer needs to be updated to reflect the new target.
1295       update_mdp_by_offset(mdp,
1296                            in_bytes(RetData::bci_displacement_offset(row)));
1297       b(profile_continue);
1298       bind(next_test);
1299     }
1300 
1301     update_mdp_for_ret(return_bci);
1302 
1303     bind(profile_continue);
1304   }
1305 }
1306 
1307 void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
1308   if (ProfileInterpreter) {
1309     Label profile_continue;
1310 
1311     // If no method data exists, go to profile_continue.
1312     test_method_data_pointer(mdp, profile_continue);
1313 
1314     set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1315 
1316     // The method data pointer needs to be updated.
1317     int mdp_delta = in_bytes(BitData::bit_data_size());
1318     if (TypeProfileCasts) {
1319       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1320     }
1321     update_mdp_by_constant(mdp, mdp_delta);
1322 
1323     bind(profile_continue);
1324   }
1325 }
1326 
1327 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2) {
1328   if (ProfileInterpreter) {
1329     Label profile_continue;
1330 
1331     // If no method data exists, go to profile_continue.
1332     test_method_data_pointer(mdp, profile_continue);
1333 
1334     // The method data pointer needs to be updated.
1335     int mdp_delta = in_bytes(BitData::bit_data_size());
1336     if (TypeProfileCasts) {
1337       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1338 
1339       // Record the object type.
1340       record_klass_in_profile(klass, mdp, reg2);
1341     }
1342     update_mdp_by_constant(mdp, mdp_delta);
1343 
1344     bind(profile_continue);
1345   }
1346 }
1347 
1348 void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
1349   if (ProfileInterpreter) {
1350     Label profile_continue;
1351 
1352     // If no method data exists, go to profile_continue.
1353     test_method_data_pointer(mdp, profile_continue);
1354 
1355     // Update the default case count
1356     increment_mdp_data_at(mdp,
1357                           in_bytes(MultiBranchData::default_count_offset()));
1358 
1359     // The method data pointer needs to be updated.
1360     update_mdp_by_offset(mdp,
1361                          in_bytes(MultiBranchData::
1362                                   default_displacement_offset()));
1363 
1364     bind(profile_continue);
1365   }
1366 }
1367 
1368 void InterpreterMacroAssembler::profile_switch_case(Register index,
1369                                                     Register mdp,
1370                                                     Register reg2) {
1371   if (ProfileInterpreter) {
1372     Label profile_continue;
1373 
1374     // If no method data exists, go to profile_continue.
1375     test_method_data_pointer(mdp, profile_continue);
1376 
1377     // Build the base (index * per_case_size_in_bytes()) +
1378     // case_array_offset_in_bytes()
1379     movw(reg2, in_bytes(MultiBranchData::per_case_size()));
1380     movw(rscratch1, in_bytes(MultiBranchData::case_array_offset()));
1381     Assembler::maddw(index, index, reg2, rscratch1);
1382 
1383     // Update the case count
1384     increment_mdp_data_at(mdp,
1385                           index,
1386                           in_bytes(MultiBranchData::relative_count_offset()));
1387 
1388     // The method data pointer needs to be updated.
1389     update_mdp_by_offset(mdp,
1390                          index,
1391                          in_bytes(MultiBranchData::
1392                                   relative_displacement_offset()));
1393 
1394     bind(profile_continue);
1395   }
1396 }
1397 
1398 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1399   if (state == atos) {
1400     MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1401   }
1402 }
1403 
1404 void InterpreterMacroAssembler::verify_FPU(int stack_depth, TosState state) { ; }
1405 
1406 
1407 void InterpreterMacroAssembler::notify_method_entry() {
1408   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1409   // track stack depth.  If it is possible to enter interp_only_mode we add
1410   // the code to check if the event should be sent.
1411   if (JvmtiExport::can_post_interpreter_events()) {
1412     Label L;
1413     ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1414     cbzw(r3, L);
1415     call_VM(noreg, CAST_FROM_FN_PTR(address,
1416                                     InterpreterRuntime::post_method_entry));
1417     bind(L);
1418   }
1419 
1420   if (DTraceMethodProbes) {
1421     get_method(c_rarg1);
1422     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
1423                  rthread, c_rarg1);
1424   }
1425 
1426   // RedefineClasses() tracing support for obsolete method entry
1427   if (log_is_enabled(Trace, redefine, class, obsolete)) {
1428     get_method(c_rarg1);
1429     call_VM_leaf(
1430       CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
1431       rthread, c_rarg1);
1432   }
1433 
1434  }
1435 
1436 
1437 void InterpreterMacroAssembler::notify_method_exit(
1438     TosState state, NotifyMethodExitMode mode) {
1439   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1440   // track stack depth.  If it is possible to enter interp_only_mode we add
1441   // the code to check if the event should be sent.
1442   if (mode == NotifyJVMTI && JvmtiExport::can_post_interpreter_events()) {
1443     Label L;
1444     // Note: frame::interpreter_frame_result has a dependency on how the
1445     // method result is saved across the call to post_method_exit. If this
1446     // is changed then the interpreter_frame_result implementation will
1447     // need to be updated too.
1448 
1449     // template interpreter will leave the result on the top of the stack.
1450     push(state);
1451     ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1452     cbz(r3, L);
1453     call_VM(noreg,
1454             CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
1455     bind(L);
1456     pop(state);
1457   }
1458 
1459   if (DTraceMethodProbes) {
1460     push(state);
1461     get_method(c_rarg1);
1462     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
1463                  rthread, c_rarg1);
1464     pop(state);
1465   }
1466 }
1467 
1468 
1469 // Jump if ((*counter_addr += increment) & mask) satisfies the condition.
1470 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
1471                                                         int increment, Address mask,
1472                                                         Register scratch, Register scratch2,
1473                                                         bool preloaded, Condition cond,
1474                                                         Label* where) {
1475   if (!preloaded) {
1476     ldrw(scratch, counter_addr);
1477   }
1478   add(scratch, scratch, increment);
1479   strw(scratch, counter_addr);
1480   ldrw(scratch2, mask);
1481   ands(scratch, scratch, scratch2);
1482   br(cond, *where);
1483 }
1484 
1485 void InterpreterMacroAssembler::call_VM_leaf_base(address entry_point,
1486                                                   int number_of_arguments) {
1487   // interpreter specific
1488   //
1489   // Note: No need to save/restore rbcp & rlocals pointer since these
1490   //       are callee saved registers and no blocking/ GC can happen
1491   //       in leaf calls.
1492 #ifdef ASSERT
1493   {
1494     Label L;
1495     ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1496     cbz(rscratch1, L);
1497     stop("InterpreterMacroAssembler::call_VM_leaf_base:"
1498          " last_sp != nullptr");
1499     bind(L);
1500   }
1501 #endif /* ASSERT */
1502   // super call
1503   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
1504 }
1505 
1506 void InterpreterMacroAssembler::call_VM_base(Register oop_result,
1507                                              Register java_thread,
1508                                              Register last_java_sp,
1509                                              address  entry_point,
1510                                              int      number_of_arguments,
1511                                              bool     check_exceptions) {
1512   // interpreter specific
1513   //
1514   // Note: Could avoid restoring locals ptr (callee saved) - however doesn't
1515   //       really make a difference for these runtime calls, since they are
1516   //       slow anyway. Btw., bcp must be saved/restored since it may change
1517   //       due to GC.
1518   // assert(java_thread == noreg , "not expecting a precomputed java thread");
1519   save_bcp();
1520 #ifdef ASSERT
1521   {
1522     Label L;
1523     ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1524     cbz(rscratch1, L);
1525     stop("InterpreterMacroAssembler::call_VM_base:"
1526          " last_sp != nullptr");
1527     bind(L);
1528   }
1529 #endif /* ASSERT */
1530   // super call
1531   MacroAssembler::call_VM_base(oop_result, noreg, last_java_sp,
1532                                entry_point, number_of_arguments,
1533                      check_exceptions);
1534 // interpreter specific
1535   restore_bcp();
1536   restore_locals();
1537 }
1538 
1539 void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result,
1540                                                     address entry_point,
1541                                                     Register arg_1) {
1542   assert(arg_1 == c_rarg1, "");
1543   Label resume_pc, not_preempted;
1544 
1545 #ifdef ASSERT
1546   {
1547     Label L;
1548     ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1549     cbz(rscratch1, L);
1550     stop("Should not have alternate return address set");
1551     bind(L);
1552   }
1553 #endif /* ASSERT */
1554 
1555   // Force freeze slow path.
1556   push_cont_fastpath();
1557 
1558   // Make VM call. In case of preemption set last_pc to the one we want to resume to.
1559   adr(rscratch1, resume_pc);
1560   str(rscratch1, Address(rthread, JavaThread::last_Java_pc_offset()));
1561   call_VM_base(oop_result, noreg, noreg, entry_point, 1, false /*check_exceptions*/);
1562 
1563   pop_cont_fastpath();
1564 
1565   // Check if preempted.
1566   ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1567   cbz(rscratch1, not_preempted);
1568   str(zr, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1569   br(rscratch1);
1570 
1571   // In case of preemption, this is where we will resume once we finally acquire the monitor.
1572   bind(resume_pc);
1573   restore_after_resume(false /* is_native */);
1574 
1575   bind(not_preempted);
1576 }
1577 
1578 void InterpreterMacroAssembler::restore_after_resume(bool is_native) {
1579   lea(rscratch1, ExternalAddress(Interpreter::cont_resume_interpreter_adapter()));
1580   blr(rscratch1);
1581   if (is_native) {
1582     // On resume we need to set up stack as expected
1583     push(dtos);
1584     push(ltos);
1585   }
1586 }
1587 
1588 void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {
1589   assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
1590   Label update, next, none;
1591 
1592   verify_oop(obj);
1593 
1594   cbnz(obj, update);
1595   orptr(mdo_addr, TypeEntries::null_seen);
1596   b(next);
1597 
1598   bind(update);
1599   load_klass(obj, obj);
1600 
1601   ldr(rscratch1, mdo_addr);
1602   eor(obj, obj, rscratch1);
1603   tst(obj, TypeEntries::type_klass_mask);
1604   br(Assembler::EQ, next); // klass seen before, nothing to
1605                            // do. The unknown bit may have been
1606                            // set already but no need to check.
1607 
1608   tbnz(obj, exact_log2(TypeEntries::type_unknown), next);
1609   // already unknown. Nothing to do anymore.
1610 
1611   cbz(rscratch1, none);
1612   cmp(rscratch1, (u1)TypeEntries::null_seen);
1613   br(Assembler::EQ, none);
1614   // There is a chance that the checks above
1615   // fail if another thread has just set the
1616   // profiling to this obj's klass
1617   eor(obj, obj, rscratch1); // get back original value before XOR
1618   ldr(rscratch1, mdo_addr);
1619   eor(obj, obj, rscratch1);
1620   tst(obj, TypeEntries::type_klass_mask);
1621   br(Assembler::EQ, next);
1622 
1623   // different than before. Cannot keep accurate profile.
1624   orptr(mdo_addr, TypeEntries::type_unknown);
1625   b(next);
1626 
1627   bind(none);
1628   // first time here. Set profile type.
1629   str(obj, mdo_addr);
1630 #ifdef ASSERT
1631   andr(obj, obj, TypeEntries::type_mask);
1632   verify_klass_ptr(obj);
1633 #endif
1634 
1635   bind(next);
1636 }
1637 
1638 void InterpreterMacroAssembler::profile_arguments_type(Register mdp, Register callee, Register tmp, bool is_virtual) {
1639   if (!ProfileInterpreter) {
1640     return;
1641   }
1642 
1643   if (MethodData::profile_arguments() || MethodData::profile_return()) {
1644     Label profile_continue;
1645 
1646     test_method_data_pointer(mdp, profile_continue);
1647 
1648     int off_to_start = is_virtual ? in_bytes(VirtualCallData::virtual_call_data_size()) : in_bytes(CounterData::counter_data_size());
1649 
1650     ldrb(rscratch1, Address(mdp, in_bytes(DataLayout::tag_offset()) - off_to_start));
1651     cmp(rscratch1, u1(is_virtual ? DataLayout::virtual_call_type_data_tag : DataLayout::call_type_data_tag));
1652     br(Assembler::NE, profile_continue);
1653 
1654     if (MethodData::profile_arguments()) {
1655       Label done;
1656       int off_to_args = in_bytes(TypeEntriesAtCall::args_data_offset());
1657 
1658       for (int i = 0; i < TypeProfileArgsLimit; i++) {
1659         if (i > 0 || MethodData::profile_return()) {
1660           // If return value type is profiled we may have no argument to profile
1661           ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1662           sub(tmp, tmp, i*TypeStackSlotEntries::per_arg_count());
1663           cmp(tmp, (u1)TypeStackSlotEntries::per_arg_count());
1664           add(rscratch1, mdp, off_to_args);
1665           br(Assembler::LT, done);
1666         }
1667         ldr(tmp, Address(callee, Method::const_offset()));
1668         load_unsigned_short(tmp, Address(tmp, ConstMethod::size_of_parameters_offset()));
1669         // stack offset o (zero based) from the start of the argument
1670         // list, for n arguments translates into offset n - o - 1 from
1671         // the end of the argument list
1672         ldr(rscratch1, Address(mdp, in_bytes(TypeEntriesAtCall::stack_slot_offset(i))));
1673         sub(tmp, tmp, rscratch1);
1674         sub(tmp, tmp, 1);
1675         Address arg_addr = argument_address(tmp);
1676         ldr(tmp, arg_addr);
1677 
1678         Address mdo_arg_addr(mdp, in_bytes(TypeEntriesAtCall::argument_type_offset(i)));
1679         profile_obj_type(tmp, mdo_arg_addr);
1680 
1681         int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
1682         off_to_args += to_add;
1683       }
1684 
1685       if (MethodData::profile_return()) {
1686         ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1687         sub(tmp, tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
1688       }
1689 
1690       add(rscratch1, mdp, off_to_args);
1691       bind(done);
1692       mov(mdp, rscratch1);
1693 
1694       if (MethodData::profile_return()) {
1695         // We're right after the type profile for the last
1696         // argument. tmp is the number of cells left in the
1697         // CallTypeData/VirtualCallTypeData to reach its end. Non null
1698         // if there's a return to profile.
1699         assert(ReturnTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
1700         add(mdp, mdp, tmp, LSL, exact_log2(DataLayout::cell_size));
1701       }
1702       str(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1703     } else {
1704       assert(MethodData::profile_return(), "either profile call args or call ret");
1705       update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
1706     }
1707 
1708     // mdp points right after the end of the
1709     // CallTypeData/VirtualCallTypeData, right after the cells for the
1710     // return value type if there's one
1711 
1712     bind(profile_continue);
1713   }
1714 }
1715 
1716 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
1717   assert_different_registers(mdp, ret, tmp, rbcp);
1718   if (ProfileInterpreter && MethodData::profile_return()) {
1719     Label profile_continue, done;
1720 
1721     test_method_data_pointer(mdp, profile_continue);
1722 
1723     if (MethodData::profile_return_jsr292_only()) {
1724       assert(Method::intrinsic_id_size_in_bytes() == 2, "assuming Method::_intrinsic_id is u2");
1725 
1726       // If we don't profile all invoke bytecodes we must make sure
1727       // it's a bytecode we indeed profile. We can't go back to the
1728       // beginning of the ProfileData we intend to update to check its
1729       // type because we're right after it and we don't known its
1730       // length
1731       Label do_profile;
1732       ldrb(rscratch1, Address(rbcp, 0));
1733       cmp(rscratch1, (u1)Bytecodes::_invokedynamic);
1734       br(Assembler::EQ, do_profile);
1735       cmp(rscratch1, (u1)Bytecodes::_invokehandle);
1736       br(Assembler::EQ, do_profile);
1737       get_method(tmp);
1738       ldrh(rscratch1, Address(tmp, Method::intrinsic_id_offset()));
1739       subs(zr, rscratch1, static_cast<int>(vmIntrinsics::_compiledLambdaForm));
1740       br(Assembler::NE, profile_continue);
1741 
1742       bind(do_profile);
1743     }
1744 
1745     Address mdo_ret_addr(mdp, -in_bytes(ReturnTypeEntry::size()));
1746     mov(tmp, ret);
1747     profile_obj_type(tmp, mdo_ret_addr);
1748 
1749     bind(profile_continue);
1750   }
1751 }
1752 
1753 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
1754   assert_different_registers(rscratch1, rscratch2, mdp, tmp1, tmp2);
1755   if (ProfileInterpreter && MethodData::profile_parameters()) {
1756     Label profile_continue, done;
1757 
1758     test_method_data_pointer(mdp, profile_continue);
1759 
1760     // Load the offset of the area within the MDO used for
1761     // parameters. If it's negative we're not profiling any parameters
1762     ldrw(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
1763     tbnz(tmp1, 31, profile_continue);  // i.e. sign bit set
1764 
1765     // Compute a pointer to the area for parameters from the offset
1766     // and move the pointer to the slot for the last
1767     // parameters. Collect profiling from last parameter down.
1768     // mdo start + parameters offset + array length - 1
1769     add(mdp, mdp, tmp1);
1770     ldr(tmp1, Address(mdp, ArrayData::array_len_offset()));
1771     sub(tmp1, tmp1, TypeStackSlotEntries::per_arg_count());
1772 
1773     Label loop;
1774     bind(loop);
1775 
1776     int off_base = in_bytes(ParametersTypeData::stack_slot_offset(0));
1777     int type_base = in_bytes(ParametersTypeData::type_offset(0));
1778     int per_arg_scale = exact_log2(DataLayout::cell_size);
1779     add(rscratch1, mdp, off_base);
1780     add(rscratch2, mdp, type_base);
1781 
1782     Address arg_off(rscratch1, tmp1, Address::lsl(per_arg_scale));
1783     Address arg_type(rscratch2, tmp1, Address::lsl(per_arg_scale));
1784 
1785     // load offset on the stack from the slot for this parameter
1786     ldr(tmp2, arg_off);
1787     neg(tmp2, tmp2);
1788     // read the parameter from the local area
1789     ldr(tmp2, Address(rlocals, tmp2, Address::lsl(Interpreter::logStackElementSize)));
1790 
1791     // profile the parameter
1792     profile_obj_type(tmp2, arg_type);
1793 
1794     // go to next parameter
1795     subs(tmp1, tmp1, TypeStackSlotEntries::per_arg_count());
1796     br(Assembler::GE, loop);
1797 
1798     bind(profile_continue);
1799   }
1800 }
1801 
1802 void InterpreterMacroAssembler::load_resolved_indy_entry(Register cache, Register index) {
1803   // Get index out of bytecode pointer, get_cache_entry_pointer_at_bcp
1804   get_cache_index_at_bcp(index, 1, sizeof(u4));
1805   // Get address of invokedynamic array
1806   ldr(cache, Address(rcpool, in_bytes(ConstantPoolCache::invokedynamic_entries_offset())));
1807   // Scale the index to be the entry index * sizeof(ResolvedIndyEntry)
1808   lsl(index, index, log2i_exact(sizeof(ResolvedIndyEntry)));
1809   add(cache, cache, Array<ResolvedIndyEntry>::base_offset_in_bytes());
1810   lea(cache, Address(cache, index));
1811 }
1812 
1813 void InterpreterMacroAssembler::load_field_entry(Register cache, Register index, int bcp_offset) {
1814   // Get index out of bytecode pointer
1815   get_cache_index_at_bcp(index, bcp_offset, sizeof(u2));
1816   // Take shortcut if the size is a power of 2
1817   if (is_power_of_2(sizeof(ResolvedFieldEntry))) {
1818     lsl(index, index, log2i_exact(sizeof(ResolvedFieldEntry))); // Scale index by power of 2
1819   } else {
1820     mov(cache, sizeof(ResolvedFieldEntry));
1821     mul(index, index, cache); // Scale the index to be the entry index * sizeof(ResolvedFieldEntry)
1822   }
1823   // Get address of field entries array
1824   ldr(cache, Address(rcpool, ConstantPoolCache::field_entries_offset()));
1825   add(cache, cache, Array<ResolvedFieldEntry>::base_offset_in_bytes());
1826   lea(cache, Address(cache, index));
1827   // Prevents stale data from being read after the bytecode is patched to the fast bytecode
1828   membar(MacroAssembler::LoadLoad);
1829 }
1830 
1831 void InterpreterMacroAssembler::load_method_entry(Register cache, Register index, int bcp_offset) {
1832   // Get index out of bytecode pointer
1833   get_cache_index_at_bcp(index, bcp_offset, sizeof(u2));
1834   mov(cache, sizeof(ResolvedMethodEntry));
1835   mul(index, index, cache); // Scale the index to be the entry index * sizeof(ResolvedMethodEntry)
1836 
1837   // Get address of field entries array
1838   ldr(cache, Address(rcpool, ConstantPoolCache::method_entries_offset()));
1839   add(cache, cache, Array<ResolvedMethodEntry>::base_offset_in_bytes());
1840   lea(cache, Address(cache, index));
1841 }