1 /*
   2  * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "compiler/compiler_globals.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/barrierSetAssembler.hpp"
  30 #include "interp_masm_aarch64.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "interpreter/interpreterRuntime.hpp"
  33 #include "logging/log.hpp"
  34 #include "oops/arrayOop.hpp"
  35 #include "oops/constMethodFlags.hpp"
  36 #include "oops/markWord.hpp"
  37 #include "oops/method.hpp"
  38 #include "oops/methodData.hpp"
  39 #include "oops/inlineKlass.hpp"
  40 #include "oops/resolvedFieldEntry.hpp"
  41 #include "oops/resolvedIndyEntry.hpp"
  42 #include "oops/resolvedMethodEntry.hpp"
  43 #include "prims/jvmtiExport.hpp"
  44 #include "prims/jvmtiThreadState.hpp"
  45 #include "runtime/basicLock.hpp"
  46 #include "runtime/frame.inline.hpp"
  47 #include "runtime/javaThread.hpp"
  48 #include "runtime/safepointMechanism.hpp"
  49 #include "runtime/sharedRuntime.hpp"
  50 #include "utilities/powerOfTwo.hpp"
  51 
  52 void InterpreterMacroAssembler::narrow(Register result) {
  53 
  54   // Get method->_constMethod->_result_type
  55   ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
  56   ldr(rscratch1, Address(rscratch1, Method::const_offset()));
  57   ldrb(rscratch1, Address(rscratch1, ConstMethod::result_type_offset()));
  58 
  59   Label done, notBool, notByte, notChar;
  60 
  61   // common case first
  62   cmpw(rscratch1, T_INT);
  63   br(Assembler::EQ, done);
  64 
  65   // mask integer result to narrower return type.
  66   cmpw(rscratch1, T_BOOLEAN);
  67   br(Assembler::NE, notBool);
  68   andw(result, result, 0x1);
  69   b(done);
  70 
  71   bind(notBool);
  72   cmpw(rscratch1, T_BYTE);
  73   br(Assembler::NE, notByte);
  74   sbfx(result, result, 0, 8);
  75   b(done);
  76 
  77   bind(notByte);
  78   cmpw(rscratch1, T_CHAR);
  79   br(Assembler::NE, notChar);
  80   ubfx(result, result, 0, 16);  // truncate upper 16 bits
  81   b(done);
  82 
  83   bind(notChar);
  84   sbfx(result, result, 0, 16);     // sign-extend short
  85 
  86   // Nothing to do for T_INT
  87   bind(done);
  88 }
  89 
  90 void InterpreterMacroAssembler::jump_to_entry(address entry) {
  91   assert(entry, "Entry must have been generated by now");
  92   b(entry);
  93 }
  94 
  95 void InterpreterMacroAssembler::check_and_handle_popframe(Register java_thread) {
  96   if (JvmtiExport::can_pop_frame()) {
  97     Label L;
  98     // Initiate popframe handling only if it is not already being
  99     // processed.  If the flag has the popframe_processing bit set, it
 100     // means that this code is called *during* popframe handling - we
 101     // don't want to reenter.
 102     // This method is only called just after the call into the vm in
 103     // call_VM_base, so the arg registers are available.
 104     ldrw(rscratch1, Address(rthread, JavaThread::popframe_condition_offset()));
 105     tbz(rscratch1, exact_log2(JavaThread::popframe_pending_bit), L);
 106     tbnz(rscratch1, exact_log2(JavaThread::popframe_processing_bit), L);
 107     // Call Interpreter::remove_activation_preserving_args_entry() to get the
 108     // address of the same-named entrypoint in the generated interpreter code.
 109     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));
 110     br(r0);
 111     bind(L);
 112   }
 113 }
 114 
 115 
 116 void InterpreterMacroAssembler::load_earlyret_value(TosState state) {
 117   ldr(r2, Address(rthread, JavaThread::jvmti_thread_state_offset()));
 118   const Address tos_addr(r2, JvmtiThreadState::earlyret_tos_offset());
 119   const Address oop_addr(r2, JvmtiThreadState::earlyret_oop_offset());
 120   const Address val_addr(r2, JvmtiThreadState::earlyret_value_offset());
 121   switch (state) {
 122     case atos: ldr(r0, oop_addr);
 123                str(zr, oop_addr);
 124                interp_verify_oop(r0, state);        break;
 125     case ltos: ldr(r0, val_addr);                   break;
 126     case btos:                                   // fall through
 127     case ztos:                                   // fall through
 128     case ctos:                                   // fall through
 129     case stos:                                   // fall through
 130     case itos: ldrw(r0, val_addr);                  break;
 131     case ftos: ldrs(v0, val_addr);                  break;
 132     case dtos: ldrd(v0, val_addr);                  break;
 133     case vtos: /* nothing to do */                  break;
 134     default  : ShouldNotReachHere();
 135   }
 136   // Clean up tos value in the thread object
 137   movw(rscratch1, (int) ilgl);
 138   strw(rscratch1, tos_addr);
 139   strw(zr, val_addr);
 140 }
 141 
 142 
 143 void InterpreterMacroAssembler::check_and_handle_earlyret(Register java_thread) {
 144   if (JvmtiExport::can_force_early_return()) {
 145     Label L;
 146     ldr(rscratch1, Address(rthread, JavaThread::jvmti_thread_state_offset()));
 147     cbz(rscratch1, L); // if (thread->jvmti_thread_state() == nullptr) exit;
 148 
 149     // Initiate earlyret handling only if it is not already being processed.
 150     // If the flag has the earlyret_processing bit set, it means that this code
 151     // is called *during* earlyret handling - we don't want to reenter.
 152     ldrw(rscratch1, Address(rscratch1, JvmtiThreadState::earlyret_state_offset()));
 153     cmpw(rscratch1, JvmtiThreadState::earlyret_pending);
 154     br(Assembler::NE, L);
 155 
 156     // Call Interpreter::remove_activation_early_entry() to get the address of the
 157     // same-named entrypoint in the generated interpreter code.
 158     ldr(rscratch1, Address(rthread, JavaThread::jvmti_thread_state_offset()));
 159     ldrw(rscratch1, Address(rscratch1, JvmtiThreadState::earlyret_tos_offset()));
 160     call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), rscratch1);
 161     br(r0);
 162     bind(L);
 163   }
 164 }
 165 
 166 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(
 167   Register reg,
 168   int bcp_offset) {
 169   assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");
 170   ldrh(reg, Address(rbcp, bcp_offset));
 171   rev16(reg, reg);
 172 }
 173 
 174 void InterpreterMacroAssembler::get_dispatch() {
 175   uint64_t offset;
 176   adrp(rdispatch, ExternalAddress((address)Interpreter::dispatch_table()), offset);
 177   // Use add() here after ARDP, rather than lea().
 178   // lea() does not generate anything if its offset is zero.
 179   // However, relocs expect to find either an ADD or a load/store
 180   // insn after an ADRP.  add() always generates an ADD insn, even
 181   // for add(Rn, Rn, 0).
 182   add(rdispatch, rdispatch, offset);
 183 }
 184 
 185 void InterpreterMacroAssembler::get_cache_index_at_bcp(Register index,
 186                                                        int bcp_offset,
 187                                                        size_t index_size) {
 188   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
 189   if (index_size == sizeof(u2)) {
 190     load_unsigned_short(index, Address(rbcp, bcp_offset));
 191   } else if (index_size == sizeof(u4)) {
 192     // assert(EnableInvokeDynamic, "giant index used only for JSR 292");
 193     ldrw(index, Address(rbcp, bcp_offset));
 194   } else if (index_size == sizeof(u1)) {
 195     load_unsigned_byte(index, Address(rbcp, bcp_offset));
 196   } else {
 197     ShouldNotReachHere();
 198   }
 199 }
 200 
 201 void InterpreterMacroAssembler::get_method_counters(Register method,
 202                                                     Register mcs, Label& skip) {
 203   Label has_counters;
 204   ldr(mcs, Address(method, Method::method_counters_offset()));
 205   cbnz(mcs, has_counters);
 206   call_VM(noreg, CAST_FROM_FN_PTR(address,
 207           InterpreterRuntime::build_method_counters), method);
 208   ldr(mcs, Address(method, Method::method_counters_offset()));
 209   cbz(mcs, skip); // No MethodCounters allocated, OutOfMemory
 210   bind(has_counters);
 211 }
 212 
 213 void InterpreterMacroAssembler::allocate_instance(Register klass, Register new_obj,
 214                                                   Register t1, Register t2,
 215                                                   bool clear_fields, Label& alloc_failed) {
 216   MacroAssembler::allocate_instance(klass, new_obj, t1, t2, clear_fields, alloc_failed);
 217   if (DTraceMethodProbes) {
 218       // Trigger dtrace event for fastpath
 219     push(atos);
 220     call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), new_obj);
 221     pop(atos);
 222   }
 223 }
 224 
 225 void InterpreterMacroAssembler::read_flat_field(Register entry,
 226                                                 Register field_index, Register field_offset,
 227                                                 Register temp, Register obj) {
 228   Label alloc_failed, done;
 229   const Register src = field_offset;
 230   const Register alloc_temp = r10;
 231   const Register dst_temp   = field_index;
 232   const Register layout_info = temp;
 233   assert_different_registers(obj, entry, field_index, field_offset, temp, alloc_temp);
 234 
 235   // Grab the inline field klass
 236   ldr(rscratch1, Address(entry, in_bytes(ResolvedFieldEntry::field_holder_offset())));
 237   inline_layout_info(rscratch1, field_index, layout_info);
 238 
 239   const Register field_klass = dst_temp;
 240   ldr(field_klass, Address(layout_info, in_bytes(InlineLayoutInfo::klass_offset())));
 241 
 242   // allocate buffer
 243   push(obj); // save holder
 244   allocate_instance(field_klass, obj, alloc_temp, rscratch2, false, alloc_failed);
 245 
 246   // Have an oop instance buffer, copy into it
 247   payload_address(obj, dst_temp, field_klass);  // danger, uses rscratch1
 248   pop(alloc_temp);             // restore holder
 249   lea(src, Address(alloc_temp, field_offset));
 250   // call_VM_leaf, clobbers a few regs, save restore new obj
 251   push(obj);
 252   flat_field_copy(IS_DEST_UNINITIALIZED, src, dst_temp, layout_info);
 253   pop(obj);
 254   b(done);
 255 
 256   bind(alloc_failed);
 257   pop(obj);
 258   call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flat_field),
 259           obj, entry);
 260 
 261   bind(done);
 262   membar(Assembler::StoreStore);
 263 }
 264 
 265 // Load object from cpool->resolved_references(index)
 266 void InterpreterMacroAssembler::load_resolved_reference_at_index(
 267                                            Register result, Register index, Register tmp) {
 268   assert_different_registers(result, index);
 269 
 270   get_constant_pool(result);
 271   // load pointer for resolved_references[] objArray
 272   ldr(result, Address(result, ConstantPool::cache_offset()));
 273   ldr(result, Address(result, ConstantPoolCache::resolved_references_offset()));
 274   resolve_oop_handle(result, tmp, rscratch2);
 275   // Add in the index
 276   add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 277   load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)), tmp, rscratch2);
 278 }
 279 
 280 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
 281                              Register cpool, Register index, Register klass, Register temp) {
 282   add(temp, cpool, index, LSL, LogBytesPerWord);
 283   ldrh(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index
 284   ldr(klass, Address(cpool,  ConstantPool::resolved_klasses_offset())); // klass = cpool->_resolved_klasses
 285   add(klass, klass, temp, LSL, LogBytesPerWord);
 286   ldr(klass, Address(klass, Array<Klass*>::base_offset_in_bytes()));
 287 }
 288 
 289 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
 290 // subtype of super_klass.
 291 //
 292 // Args:
 293 //      r0: superklass
 294 //      Rsub_klass: subklass
 295 //
 296 // Kills:
 297 //      r2, r5
 298 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
 299                                                   Label& ok_is_subtype,
 300                                                   bool profile) {
 301   assert(Rsub_klass != r0, "r0 holds superklass");
 302   assert(Rsub_klass != r2, "r2 holds 2ndary super array length");
 303   assert(Rsub_klass != r5, "r5 holds 2ndary super array scan ptr");
 304 
 305   // Profile the not-null value's klass.
 306   if (profile) {
 307     profile_typecheck(r2, Rsub_klass, r5); // blows r2, reloads r5
 308   }
 309 
 310   // Do the check.
 311   check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2
 312 }
 313 
 314 // Java Expression Stack
 315 
 316 void InterpreterMacroAssembler::pop_ptr(Register r) {
 317   ldr(r, post(esp, wordSize));
 318 }
 319 
 320 void InterpreterMacroAssembler::pop_i(Register r) {
 321   ldrw(r, post(esp, wordSize));
 322 }
 323 
 324 void InterpreterMacroAssembler::pop_l(Register r) {
 325   ldr(r, post(esp, 2 * Interpreter::stackElementSize));
 326 }
 327 
 328 void InterpreterMacroAssembler::push_ptr(Register r) {
 329   str(r, pre(esp, -wordSize));
 330  }
 331 
 332 void InterpreterMacroAssembler::push_i(Register r) {
 333   str(r, pre(esp, -wordSize));
 334 }
 335 
 336 void InterpreterMacroAssembler::push_l(Register r) {
 337   str(zr, pre(esp, -wordSize));
 338   str(r, pre(esp, - wordSize));
 339 }
 340 
 341 void InterpreterMacroAssembler::pop_f(FloatRegister r) {
 342   ldrs(r, post(esp, wordSize));
 343 }
 344 
 345 void InterpreterMacroAssembler::pop_d(FloatRegister r) {
 346   ldrd(r, post(esp, 2 * Interpreter::stackElementSize));
 347 }
 348 
 349 void InterpreterMacroAssembler::push_f(FloatRegister r) {
 350   strs(r, pre(esp, -wordSize));
 351 }
 352 
 353 void InterpreterMacroAssembler::push_d(FloatRegister r) {
 354   strd(r, pre(esp, 2* -wordSize));
 355 }
 356 
 357 void InterpreterMacroAssembler::pop(TosState state) {
 358   switch (state) {
 359   case atos: pop_ptr();                 break;
 360   case btos:
 361   case ztos:
 362   case ctos:
 363   case stos:
 364   case itos: pop_i();                   break;
 365   case ltos: pop_l();                   break;
 366   case ftos: pop_f();                   break;
 367   case dtos: pop_d();                   break;
 368   case vtos: /* nothing to do */        break;
 369   default:   ShouldNotReachHere();
 370   }
 371   interp_verify_oop(r0, state);
 372 }
 373 
 374 void InterpreterMacroAssembler::push(TosState state) {
 375   interp_verify_oop(r0, state);
 376   switch (state) {
 377   case atos: push_ptr();                break;
 378   case btos:
 379   case ztos:
 380   case ctos:
 381   case stos:
 382   case itos: push_i();                  break;
 383   case ltos: push_l();                  break;
 384   case ftos: push_f();                  break;
 385   case dtos: push_d();                  break;
 386   case vtos: /* nothing to do */        break;
 387   default  : ShouldNotReachHere();
 388   }
 389 }
 390 
 391 // Helpers for swap and dup
 392 void InterpreterMacroAssembler::load_ptr(int n, Register val) {
 393   ldr(val, Address(esp, Interpreter::expr_offset_in_bytes(n)));
 394 }
 395 
 396 void InterpreterMacroAssembler::store_ptr(int n, Register val) {
 397   str(val, Address(esp, Interpreter::expr_offset_in_bytes(n)));
 398 }
 399 
 400 void InterpreterMacroAssembler::load_float(Address src) {
 401   ldrs(v0, src);
 402 }
 403 
 404 void InterpreterMacroAssembler::load_double(Address src) {
 405   ldrd(v0, src);
 406 }
 407 
 408 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() {
 409   // set sender sp
 410   mov(r19_sender_sp, sp);
 411   // record last_sp
 412   sub(rscratch1, esp, rfp);
 413   asr(rscratch1, rscratch1, Interpreter::logStackElementSize);
 414   str(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
 415 }
 416 
 417 // Jump to from_interpreted entry of a call unless single stepping is possible
 418 // in this thread in which case we must call the i2i entry
 419 void InterpreterMacroAssembler::jump_from_interpreted(Register method, Register temp) {
 420   prepare_to_jump_from_interpreted();
 421 
 422   if (JvmtiExport::can_post_interpreter_events()) {
 423     Label run_compiled_code;
 424     // JVMTI events, such as single-stepping, are implemented partly by avoiding running
 425     // compiled code in threads for which the event is enabled.  Check here for
 426     // interp_only_mode if these events CAN be enabled.
 427     ldrw(rscratch1, Address(rthread, JavaThread::interp_only_mode_offset()));
 428     cbzw(rscratch1, run_compiled_code);
 429     ldr(rscratch1, Address(method, Method::interpreter_entry_offset()));
 430     br(rscratch1);
 431     bind(run_compiled_code);
 432   }
 433 
 434   ldr(rscratch1, Address(method, Method::from_interpreted_offset()));
 435   br(rscratch1);
 436 }
 437 
 438 // The following two routines provide a hook so that an implementation
 439 // can schedule the dispatch in two parts.  amd64 does not do this.
 440 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) {
 441 }
 442 
 443 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {
 444     dispatch_next(state, step);
 445 }
 446 
 447 void InterpreterMacroAssembler::dispatch_base(TosState state,
 448                                               address* table,
 449                                               bool verifyoop,
 450                                               bool generate_poll) {
 451   if (VerifyActivationFrameSize) {
 452     Label L;
 453     sub(rscratch2, rfp, esp);
 454     int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize;
 455     subs(rscratch2, rscratch2, min_frame_size);
 456     br(Assembler::GE, L);
 457     stop("broken stack frame");
 458     bind(L);
 459   }
 460   if (verifyoop) {
 461     interp_verify_oop(r0, state);
 462   }
 463 
 464   Label safepoint;
 465   address* const safepoint_table = Interpreter::safept_table(state);
 466   bool needs_thread_local_poll = generate_poll && table != safepoint_table;
 467 
 468   if (needs_thread_local_poll) {
 469     NOT_PRODUCT(block_comment("Thread-local Safepoint poll"));
 470     ldr(rscratch2, Address(rthread, JavaThread::polling_word_offset()));
 471     tbnz(rscratch2, exact_log2(SafepointMechanism::poll_bit()), safepoint);
 472   }
 473 
 474   if (table == Interpreter::dispatch_table(state)) {
 475     addw(rscratch2, rscratch1, Interpreter::distance_from_dispatch_table(state));
 476     ldr(rscratch2, Address(rdispatch, rscratch2, Address::uxtw(3)));
 477   } else {
 478     mov(rscratch2, (address)table);
 479     ldr(rscratch2, Address(rscratch2, rscratch1, Address::uxtw(3)));
 480   }
 481   br(rscratch2);
 482 
 483   if (needs_thread_local_poll) {
 484     bind(safepoint);
 485     lea(rscratch2, ExternalAddress((address)safepoint_table));
 486     ldr(rscratch2, Address(rscratch2, rscratch1, Address::uxtw(3)));
 487     br(rscratch2);
 488   }
 489 }
 490 
 491 void InterpreterMacroAssembler::dispatch_only(TosState state, bool generate_poll) {
 492   dispatch_base(state, Interpreter::dispatch_table(state), true, generate_poll);
 493 }
 494 
 495 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) {
 496   dispatch_base(state, Interpreter::normal_table(state));
 497 }
 498 
 499 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
 500   dispatch_base(state, Interpreter::normal_table(state), false);
 501 }
 502 
 503 
 504 void InterpreterMacroAssembler::dispatch_next(TosState state, int step, bool generate_poll) {
 505   // load next bytecode
 506   ldrb(rscratch1, Address(pre(rbcp, step)));
 507   dispatch_base(state, Interpreter::dispatch_table(state), /*verifyoop*/true, generate_poll);
 508 }
 509 
 510 void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) {
 511   // load current bytecode
 512   ldrb(rscratch1, Address(rbcp, 0));
 513   dispatch_base(state, table);
 514 }
 515 
 516 // remove activation
 517 //
 518 // Apply stack watermark barrier.
 519 // Unlock the receiver if this is a synchronized method.
 520 // Unlock any Java monitors from synchronized blocks.
 521 // Remove the activation from the stack.
 522 //
 523 // If there are locked Java monitors
 524 //    If throw_monitor_exception
 525 //       throws IllegalMonitorStateException
 526 //    Else if install_monitor_exception
 527 //       installs IllegalMonitorStateException
 528 //    Else
 529 //       no error processing
 530 void InterpreterMacroAssembler::remove_activation(
 531         TosState state,
 532         bool throw_monitor_exception,
 533         bool install_monitor_exception,
 534         bool notify_jvmdi) {
 535   // Note: Registers r3 xmm0 may be in use for the
 536   // result check if synchronized method
 537   Label unlocked, unlock, no_unlock;
 538 
 539   // The below poll is for the stack watermark barrier. It allows fixing up frames lazily,
 540   // that would normally not be safe to use. Such bad returns into unsafe territory of
 541   // the stack, will call InterpreterRuntime::at_unwind.
 542   Label slow_path;
 543   Label fast_path;
 544   safepoint_poll(slow_path, true /* at_return */, false /* acquire */, false /* in_nmethod */);
 545   br(Assembler::AL, fast_path);
 546   bind(slow_path);
 547   push(state);
 548   set_last_Java_frame(esp, rfp, (address)pc(), rscratch1);
 549   super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
 550   reset_last_Java_frame(true);
 551   pop(state);
 552   bind(fast_path);
 553 
 554   // get the value of _do_not_unlock_if_synchronized into r3
 555   const Address do_not_unlock_if_synchronized(rthread,
 556     in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
 557   ldrb(r3, do_not_unlock_if_synchronized);
 558   strb(zr, do_not_unlock_if_synchronized); // reset the flag
 559 
 560  // get method access flags
 561   ldr(r1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
 562   ldrh(r2, Address(r1, Method::access_flags_offset()));
 563   tbz(r2, exact_log2(JVM_ACC_SYNCHRONIZED), unlocked);
 564 
 565   // Don't unlock anything if the _do_not_unlock_if_synchronized flag
 566   // is set.
 567   cbnz(r3, no_unlock);
 568 
 569   // unlock monitor
 570   push(state); // save result
 571 
 572   // BasicObjectLock will be first in list, since this is a
 573   // synchronized method. However, need to check that the object has
 574   // not been unlocked by an explicit monitorexit bytecode.
 575   const Address monitor(rfp, frame::interpreter_frame_initial_sp_offset *
 576                         wordSize - (int) sizeof(BasicObjectLock));
 577   // We use c_rarg1 so that if we go slow path it will be the correct
 578   // register for unlock_object to pass to VM directly
 579   lea(c_rarg1, monitor); // address of first monitor
 580 
 581   ldr(r0, Address(c_rarg1, BasicObjectLock::obj_offset()));
 582   cbnz(r0, unlock);
 583 
 584   pop(state);
 585   if (throw_monitor_exception) {
 586     // Entry already unlocked, need to throw exception
 587     call_VM(noreg, CAST_FROM_FN_PTR(address,
 588                    InterpreterRuntime::throw_illegal_monitor_state_exception));
 589     should_not_reach_here();
 590   } else {
 591     // Monitor already unlocked during a stack unroll. If requested,
 592     // install an illegal_monitor_state_exception.  Continue with
 593     // stack unrolling.
 594     if (install_monitor_exception) {
 595       call_VM(noreg, CAST_FROM_FN_PTR(address,
 596                      InterpreterRuntime::new_illegal_monitor_state_exception));
 597     }
 598     b(unlocked);
 599   }
 600 
 601   bind(unlock);
 602   unlock_object(c_rarg1);
 603   pop(state);
 604 
 605   // Check that for block-structured locking (i.e., that all locked
 606   // objects has been unlocked)
 607   bind(unlocked);
 608 
 609   // r0: Might contain return value
 610 
 611   // Check that all monitors are unlocked
 612   {
 613     Label loop, exception, entry, restart;
 614     const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
 615     const Address monitor_block_top(
 616         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
 617     const Address monitor_block_bot(
 618         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
 619 
 620     bind(restart);
 621     // We use c_rarg1 so that if we go slow path it will be the correct
 622     // register for unlock_object to pass to VM directly
 623     ldr(c_rarg1, monitor_block_top); // derelativize pointer
 624     lea(c_rarg1, Address(rfp, c_rarg1, Address::lsl(Interpreter::logStackElementSize)));
 625     // c_rarg1 points to current entry, starting with top-most entry
 626 
 627     lea(r19, monitor_block_bot);  // points to word before bottom of
 628                                   // monitor block
 629     b(entry);
 630 
 631     // Entry already locked, need to throw exception
 632     bind(exception);
 633 
 634     if (throw_monitor_exception) {
 635       // Throw exception
 636       MacroAssembler::call_VM(noreg,
 637                               CAST_FROM_FN_PTR(address, InterpreterRuntime::
 638                                    throw_illegal_monitor_state_exception));
 639       should_not_reach_here();
 640     } else {
 641       // Stack unrolling. Unlock object and install illegal_monitor_exception.
 642       // Unlock does not block, so don't have to worry about the frame.
 643       // We don't have to preserve c_rarg1 since we are going to throw an exception.
 644 
 645       push(state);
 646       unlock_object(c_rarg1);
 647       pop(state);
 648 
 649       if (install_monitor_exception) {
 650         call_VM(noreg, CAST_FROM_FN_PTR(address,
 651                                         InterpreterRuntime::
 652                                         new_illegal_monitor_state_exception));
 653       }
 654 
 655       b(restart);
 656     }
 657 
 658     bind(loop);
 659     // check if current entry is used
 660     ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset()));
 661     cbnz(rscratch1, exception);
 662 
 663     add(c_rarg1, c_rarg1, entry_size); // otherwise advance to next entry
 664     bind(entry);
 665     cmp(c_rarg1, r19); // check if bottom reached
 666     br(Assembler::NE, loop); // if not at bottom then check this entry
 667   }
 668 
 669   bind(no_unlock);
 670 
 671   // jvmti support
 672   if (notify_jvmdi) {
 673     notify_method_exit(state, NotifyJVMTI);    // preserve TOSCA
 674   } else {
 675     notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
 676   }
 677 
 678   // remove activation
 679   // get sender esp
 680   ldr(rscratch2,
 681       Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
 682 
 683   if (StackReservedPages > 0) {
 684     // testing if reserved zone needs to be re-enabled
 685     Label no_reserved_zone_enabling;
 686 
 687     // check if already enabled - if so no re-enabling needed
 688     assert(sizeof(StackOverflow::StackGuardState) == 4, "unexpected size");
 689     ldrw(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
 690     cmpw(rscratch1, (u1)StackOverflow::stack_guard_enabled);
 691     br(Assembler::EQ, no_reserved_zone_enabling);
 692 
 693     // look for an overflow into the stack reserved zone, i.e.
 694     // interpreter_frame_sender_sp <= JavaThread::reserved_stack_activation
 695     ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
 696     cmp(rscratch2, rscratch1);
 697     br(Assembler::LS, no_reserved_zone_enabling);
 698 
 699     call_VM_leaf(
 700       CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
 701     call_VM(noreg, CAST_FROM_FN_PTR(address,
 702                    InterpreterRuntime::throw_delayed_StackOverflowError));
 703     should_not_reach_here();
 704 
 705     bind(no_reserved_zone_enabling);
 706   }
 707 
 708   if (state == atos && InlineTypeReturnedAsFields) {
 709     // Check if we are returning an non-null inline type and load its fields into registers
 710     Label skip;
 711     test_oop_is_not_inline_type(r0, rscratch2, skip);
 712 
 713     // Load fields from a buffered value with an inline class specific handler
 714     load_klass(rscratch1 /*dst*/, r0 /*src*/);
 715     ldr(rscratch1, Address(rscratch1, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 716     ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
 717     // Unpack handler can be null if inline type is not scalarizable in returns
 718     cbz(rscratch1, skip);
 719 
 720     blr(rscratch1);
 721 #ifdef ASSERT
 722     // TODO 8284443 Enable
 723     if (StressCallingConvention && false) {
 724       Label skip_stress;
 725       ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
 726       ldrw(rscratch1, Address(rscratch1, Method::flags_offset()));
 727       tstw(rscratch1, MethodFlags::has_scalarized_return_flag());
 728       br(Assembler::EQ, skip_stress);
 729       load_klass(r0, r0);
 730       orr(r0, r0, 1);
 731       bind(skip_stress);
 732     }
 733 #endif
 734     bind(skip);
 735     // Check above kills sender esp in rscratch2. Reload it.
 736     ldr(rscratch2, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
 737   }
 738 
 739   // restore sender esp
 740   mov(esp, rscratch2);
 741   // remove frame anchor
 742   leave();
 743   // If we're returning to interpreted code we will shortly be
 744   // adjusting SP to allow some space for ESP.  If we're returning to
 745   // compiled code the saved sender SP was saved in sender_sp, so this
 746   // restores it.
 747   andr(sp, esp, -16);
 748 }
 749 
 750 // Lock object
 751 //
 752 // Args:
 753 //      c_rarg1: BasicObjectLock to be used for locking
 754 //
 755 // Kills:
 756 //      r0
 757 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, .. (param regs)
 758 //      rscratch1, rscratch2 (scratch regs)
 759 void InterpreterMacroAssembler::lock_object(Register lock_reg)
 760 {
 761   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
 762   if (LockingMode == LM_MONITOR) {
 763     call_VM_preemptable(noreg,
 764             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
 765             lock_reg);
 766   } else {
 767     Label count, done;
 768 
 769     const Register swap_reg = r0;
 770     const Register tmp = c_rarg2;
 771     const Register obj_reg = c_rarg3; // Will contain the oop
 772     const Register tmp2 = c_rarg4;
 773     const Register tmp3 = c_rarg5;
 774 
 775     const int obj_offset = in_bytes(BasicObjectLock::obj_offset());
 776     const int lock_offset = in_bytes(BasicObjectLock::lock_offset());
 777     const int mark_offset = lock_offset +
 778                             BasicLock::displaced_header_offset_in_bytes();
 779 
 780     Label slow_case;
 781 
 782     // Load object pointer into obj_reg %c_rarg3
 783     ldr(obj_reg, Address(lock_reg, obj_offset));
 784 
 785     if (LockingMode == LM_LIGHTWEIGHT) {
 786       lightweight_lock(lock_reg, obj_reg, tmp, tmp2, tmp3, slow_case);
 787       b(done);
 788     } else if (LockingMode == LM_LEGACY) {
 789 
 790       if (DiagnoseSyncOnValueBasedClasses != 0) {
 791         load_klass(tmp, obj_reg);
 792         ldrb(tmp, Address(tmp, Klass::misc_flags_offset()));
 793         tst(tmp, KlassFlags::_misc_is_value_based_class);
 794         br(Assembler::NE, slow_case);
 795       }
 796 
 797       // Load (object->mark() | 1) into swap_reg
 798       ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
 799       orr(swap_reg, rscratch1, 1);
 800       if (EnableValhalla) {
 801         // Mask inline_type bit such that we go to the slow path if object is an inline type
 802         andr(swap_reg, swap_reg, ~((int) markWord::inline_type_bit_in_place));
 803       }
 804 
 805       // Save (object->mark() | 1) into BasicLock's displaced header
 806       str(swap_reg, Address(lock_reg, mark_offset));
 807 
 808       assert(lock_offset == 0,
 809              "displached header must be first word in BasicObjectLock");
 810 
 811       Label fail;
 812       cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
 813 
 814       // Fast check for recursive lock.
 815       //
 816       // Can apply the optimization only if this is a stack lock
 817       // allocated in this thread. For efficiency, we can focus on
 818       // recently allocated stack locks (instead of reading the stack
 819       // base and checking whether 'mark' points inside the current
 820       // thread stack):
 821       //  1) (mark & 7) == 0, and
 822       //  2) sp <= mark < mark + os::pagesize()
 823       //
 824       // Warning: sp + os::pagesize can overflow the stack base. We must
 825       // neither apply the optimization for an inflated lock allocated
 826       // just above the thread stack (this is why condition 1 matters)
 827       // nor apply the optimization if the stack lock is inside the stack
 828       // of another thread. The latter is avoided even in case of overflow
 829       // because we have guard pages at the end of all stacks. Hence, if
 830       // we go over the stack base and hit the stack of another thread,
 831       // this should not be in a writeable area that could contain a
 832       // stack lock allocated by that thread. As a consequence, a stack
 833       // lock less than page size away from sp is guaranteed to be
 834       // owned by the current thread.
 835       //
 836       // These 3 tests can be done by evaluating the following
 837       // expression: ((mark - sp) & (7 - os::vm_page_size())),
 838       // assuming both stack pointer and pagesize have their
 839       // least significant 3 bits clear.
 840       // NOTE: the mark is in swap_reg %r0 as the result of cmpxchg
 841       // NOTE2: aarch64 does not like to subtract sp from rn so take a
 842       // copy
 843       mov(rscratch1, sp);
 844       sub(swap_reg, swap_reg, rscratch1);
 845       ands(swap_reg, swap_reg, (uint64_t)(7 - (int)os::vm_page_size()));
 846 
 847       // Save the test result, for recursive case, the result is zero
 848       str(swap_reg, Address(lock_reg, mark_offset));
 849       br(Assembler::NE, slow_case);
 850 
 851       bind(count);
 852       inc_held_monitor_count(rscratch1);
 853       b(done);
 854     }
 855     bind(slow_case);
 856 
 857     // Call the runtime routine for slow case
 858     call_VM_preemptable(noreg,
 859             CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
 860             lock_reg);
 861 
 862     bind(done);
 863   }
 864 }
 865 
 866 
 867 // Unlocks an object. Used in monitorexit bytecode and
 868 // remove_activation.  Throws an IllegalMonitorException if object is
 869 // not locked by current thread.
 870 //
 871 // Args:
 872 //      c_rarg1: BasicObjectLock for lock
 873 //
 874 // Kills:
 875 //      r0
 876 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
 877 //      rscratch1, rscratch2 (scratch regs)
 878 void InterpreterMacroAssembler::unlock_object(Register lock_reg)
 879 {
 880   assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
 881 
 882   if (LockingMode == LM_MONITOR) {
 883     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
 884   } else {
 885     Label count, done;
 886 
 887     const Register swap_reg   = r0;
 888     const Register header_reg = c_rarg2;  // Will contain the old oopMark
 889     const Register obj_reg    = c_rarg3;  // Will contain the oop
 890     const Register tmp_reg    = c_rarg4;  // Temporary used by lightweight_unlock
 891 
 892     save_bcp(); // Save in case of exception
 893 
 894     if (LockingMode != LM_LIGHTWEIGHT) {
 895       // Convert from BasicObjectLock structure to object and BasicLock
 896       // structure Store the BasicLock address into %r0
 897       lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset()));
 898     }
 899 
 900     // Load oop into obj_reg(%c_rarg3)
 901     ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset()));
 902 
 903     // Free entry
 904     str(zr, Address(lock_reg, BasicObjectLock::obj_offset()));
 905 
 906     Label slow_case;
 907     if (LockingMode == LM_LIGHTWEIGHT) {
 908       lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
 909       b(done);
 910     } else if (LockingMode == LM_LEGACY) {
 911       // Load the old header from BasicLock structure
 912       ldr(header_reg, Address(swap_reg,
 913                               BasicLock::displaced_header_offset_in_bytes()));
 914 
 915       // Test for recursion
 916       cbz(header_reg, count);
 917 
 918       // Atomic swap back the old header
 919       cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, count, &slow_case);
 920 
 921       bind(count);
 922       dec_held_monitor_count(rscratch1);
 923       b(done);
 924     }
 925 
 926     bind(slow_case);
 927     // Call the runtime routine for slow case.
 928     str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset())); // restore obj
 929     call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
 930     bind(done);
 931     restore_bcp();
 932   }
 933 }
 934 
 935 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
 936                                                          Label& zero_continue) {
 937   assert(ProfileInterpreter, "must be profiling interpreter");
 938   ldr(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
 939   cbz(mdp, zero_continue);
 940 }
 941 
 942 // Set the method data pointer for the current bcp.
 943 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
 944   assert(ProfileInterpreter, "must be profiling interpreter");
 945   Label set_mdp;
 946   stp(r0, r1, Address(pre(sp, -2 * wordSize)));
 947 
 948   // Test MDO to avoid the call if it is null.
 949   ldr(r0, Address(rmethod, in_bytes(Method::method_data_offset())));
 950   cbz(r0, set_mdp);
 951   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rmethod, rbcp);
 952   // r0: mdi
 953   // mdo is guaranteed to be non-zero here, we checked for it before the call.
 954   ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
 955   lea(r1, Address(r1, in_bytes(MethodData::data_offset())));
 956   add(r0, r1, r0);
 957   str(r0, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
 958   bind(set_mdp);
 959   ldp(r0, r1, Address(post(sp, 2 * wordSize)));
 960 }
 961 
 962 void InterpreterMacroAssembler::verify_method_data_pointer() {
 963   assert(ProfileInterpreter, "must be profiling interpreter");
 964 #ifdef ASSERT
 965   Label verify_continue;
 966   stp(r0, r1, Address(pre(sp, -2 * wordSize)));
 967   stp(r2, r3, Address(pre(sp, -2 * wordSize)));
 968   test_method_data_pointer(r3, verify_continue); // If mdp is zero, continue
 969   get_method(r1);
 970 
 971   // If the mdp is valid, it will point to a DataLayout header which is
 972   // consistent with the bcp.  The converse is highly probable also.
 973   ldrsh(r2, Address(r3, in_bytes(DataLayout::bci_offset())));
 974   ldr(rscratch1, Address(r1, Method::const_offset()));
 975   add(r2, r2, rscratch1, Assembler::LSL);
 976   lea(r2, Address(r2, ConstMethod::codes_offset()));
 977   cmp(r2, rbcp);
 978   br(Assembler::EQ, verify_continue);
 979   // r1: method
 980   // rbcp: bcp // rbcp == 22
 981   // r3: mdp
 982   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp),
 983                r1, rbcp, r3);
 984   bind(verify_continue);
 985   ldp(r2, r3, Address(post(sp, 2 * wordSize)));
 986   ldp(r0, r1, Address(post(sp, 2 * wordSize)));
 987 #endif // ASSERT
 988 }
 989 
 990 
 991 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in,
 992                                                 int constant,
 993                                                 Register value) {
 994   assert(ProfileInterpreter, "must be profiling interpreter");
 995   Address data(mdp_in, constant);
 996   str(value, data);
 997 }
 998 
 999 
1000 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
1001                                                       int constant,
1002                                                       bool decrement) {
1003   increment_mdp_data_at(mdp_in, noreg, constant, decrement);
1004 }
1005 
1006 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
1007                                                       Register reg,
1008                                                       int constant,
1009                                                       bool decrement) {
1010   assert(ProfileInterpreter, "must be profiling interpreter");
1011   // %%% this does 64bit counters at best it is wasting space
1012   // at worst it is a rare bug when counters overflow
1013 
1014   assert_different_registers(rscratch2, rscratch1, mdp_in, reg);
1015 
1016   Address addr1(mdp_in, constant);
1017   Address addr2(rscratch2, reg, Address::lsl(0));
1018   Address &addr = addr1;
1019   if (reg != noreg) {
1020     lea(rscratch2, addr1);
1021     addr = addr2;
1022   }
1023 
1024   if (decrement) {
1025     // Decrement the register.  Set condition codes.
1026     // Intel does this
1027     // addptr(data, (int32_t) -DataLayout::counter_increment);
1028     // If the decrement causes the counter to overflow, stay negative
1029     // Label L;
1030     // jcc(Assembler::negative, L);
1031     // addptr(data, (int32_t) DataLayout::counter_increment);
1032     // so we do this
1033     ldr(rscratch1, addr);
1034     subs(rscratch1, rscratch1, (unsigned)DataLayout::counter_increment);
1035     Label L;
1036     br(Assembler::LO, L);       // skip store if counter underflow
1037     str(rscratch1, addr);
1038     bind(L);
1039   } else {
1040     assert(DataLayout::counter_increment == 1,
1041            "flow-free idiom only works with 1");
1042     // Intel does this
1043     // Increment the register.  Set carry flag.
1044     // addptr(data, DataLayout::counter_increment);
1045     // If the increment causes the counter to overflow, pull back by 1.
1046     // sbbptr(data, (int32_t)0);
1047     // so we do this
1048     ldr(rscratch1, addr);
1049     adds(rscratch1, rscratch1, DataLayout::counter_increment);
1050     Label L;
1051     br(Assembler::CS, L);       // skip store if counter overflow
1052     str(rscratch1, addr);
1053     bind(L);
1054   }
1055 }
1056 
1057 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in,
1058                                                 int flag_byte_constant) {
1059   assert(ProfileInterpreter, "must be profiling interpreter");
1060   int flags_offset = in_bytes(DataLayout::flags_offset());
1061   // Set the flag
1062   ldrb(rscratch1, Address(mdp_in, flags_offset));
1063   orr(rscratch1, rscratch1, flag_byte_constant);
1064   strb(rscratch1, Address(mdp_in, flags_offset));
1065 }
1066 
1067 
1068 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in,
1069                                                  int offset,
1070                                                  Register value,
1071                                                  Register test_value_out,
1072                                                  Label& not_equal_continue) {
1073   assert(ProfileInterpreter, "must be profiling interpreter");
1074   if (test_value_out == noreg) {
1075     ldr(rscratch1, Address(mdp_in, offset));
1076     cmp(value, rscratch1);
1077   } else {
1078     // Put the test value into a register, so caller can use it:
1079     ldr(test_value_out, Address(mdp_in, offset));
1080     cmp(value, test_value_out);
1081   }
1082   br(Assembler::NE, not_equal_continue);
1083 }
1084 
1085 
1086 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in,
1087                                                      int offset_of_disp) {
1088   assert(ProfileInterpreter, "must be profiling interpreter");
1089   ldr(rscratch1, Address(mdp_in, offset_of_disp));
1090   add(mdp_in, mdp_in, rscratch1, LSL);
1091   str(mdp_in, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1092 }
1093 
1094 
1095 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in,
1096                                                      Register reg,
1097                                                      int offset_of_disp) {
1098   assert(ProfileInterpreter, "must be profiling interpreter");
1099   lea(rscratch1, Address(mdp_in, offset_of_disp));
1100   ldr(rscratch1, Address(rscratch1, reg, Address::lsl(0)));
1101   add(mdp_in, mdp_in, rscratch1, LSL);
1102   str(mdp_in, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1103 }
1104 
1105 
1106 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in,
1107                                                        int constant) {
1108   assert(ProfileInterpreter, "must be profiling interpreter");
1109   add(mdp_in, mdp_in, (unsigned)constant);
1110   str(mdp_in, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1111 }
1112 
1113 
1114 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {
1115   assert(ProfileInterpreter, "must be profiling interpreter");
1116   // save/restore across call_VM
1117   stp(zr, return_bci, Address(pre(sp, -2 * wordSize)));
1118   call_VM(noreg,
1119           CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret),
1120           return_bci);
1121   ldp(zr, return_bci, Address(post(sp, 2 * wordSize)));
1122 }
1123 
1124 
1125 void InterpreterMacroAssembler::profile_taken_branch(Register mdp,
1126                                                      Register bumped_count) {
1127   if (ProfileInterpreter) {
1128     Label profile_continue;
1129 
1130     // If no method data exists, go to profile_continue.
1131     // Otherwise, assign to mdp
1132     test_method_data_pointer(mdp, profile_continue);
1133 
1134     // We are taking a branch.  Increment the taken count.
1135     // We inline increment_mdp_data_at to return bumped_count in a register
1136     //increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
1137     Address data(mdp, in_bytes(JumpData::taken_offset()));
1138     ldr(bumped_count, data);
1139     assert(DataLayout::counter_increment == 1,
1140             "flow-free idiom only works with 1");
1141     // Intel does this to catch overflow
1142     // addptr(bumped_count, DataLayout::counter_increment);
1143     // sbbptr(bumped_count, 0);
1144     // so we do this
1145     adds(bumped_count, bumped_count, DataLayout::counter_increment);
1146     Label L;
1147     br(Assembler::CS, L);       // skip store if counter overflow
1148     str(bumped_count, data);
1149     bind(L);
1150     // The method data pointer needs to be updated to reflect the new target.
1151     update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
1152     bind(profile_continue);
1153   }
1154 }
1155 
1156 
1157 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp, bool acmp) {
1158   if (ProfileInterpreter) {
1159     Label profile_continue;
1160 
1161     // If no method data exists, go to profile_continue.
1162     test_method_data_pointer(mdp, profile_continue);
1163 
1164     // We are taking a branch.  Increment the not taken count.
1165     increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
1166 
1167     // The method data pointer needs to be updated to correspond to
1168     // the next bytecode
1169     update_mdp_by_constant(mdp, acmp ? in_bytes(ACmpData::acmp_data_size()) : in_bytes(BranchData::branch_data_size()));
1170     bind(profile_continue);
1171   }
1172 }
1173 
1174 
1175 void InterpreterMacroAssembler::profile_call(Register mdp) {
1176   if (ProfileInterpreter) {
1177     Label profile_continue;
1178 
1179     // If no method data exists, go to profile_continue.
1180     test_method_data_pointer(mdp, profile_continue);
1181 
1182     // We are making a call.  Increment the count.
1183     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1184 
1185     // The method data pointer needs to be updated to reflect the new target.
1186     update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1187     bind(profile_continue);
1188   }
1189 }
1190 
1191 void InterpreterMacroAssembler::profile_final_call(Register mdp) {
1192   if (ProfileInterpreter) {
1193     Label profile_continue;
1194 
1195     // If no method data exists, go to profile_continue.
1196     test_method_data_pointer(mdp, profile_continue);
1197 
1198     // We are making a call.  Increment the count.
1199     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1200 
1201     // The method data pointer needs to be updated to reflect the new target.
1202     update_mdp_by_constant(mdp,
1203                            in_bytes(VirtualCallData::
1204                                     virtual_call_data_size()));
1205     bind(profile_continue);
1206   }
1207 }
1208 
1209 
1210 void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
1211                                                      Register mdp,
1212                                                      Register reg2,
1213                                                      bool receiver_can_be_null) {
1214   if (ProfileInterpreter) {
1215     Label profile_continue;
1216 
1217     // If no method data exists, go to profile_continue.
1218     test_method_data_pointer(mdp, profile_continue);
1219 
1220     Label skip_receiver_profile;
1221     if (receiver_can_be_null) {
1222       Label not_null;
1223       // We are making a call.  Increment the count for null receiver.
1224       increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1225       b(skip_receiver_profile);
1226       bind(not_null);
1227     }
1228 
1229     // Record the receiver type.
1230     record_klass_in_profile(receiver, mdp, reg2);
1231     bind(skip_receiver_profile);
1232 
1233     // The method data pointer needs to be updated to reflect the new target.
1234     update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1235     bind(profile_continue);
1236   }
1237 }
1238 
1239 // This routine creates a state machine for updating the multi-row
1240 // type profile at a virtual call site (or other type-sensitive bytecode).
1241 // The machine visits each row (of receiver/count) until the receiver type
1242 // is found, or until it runs out of rows.  At the same time, it remembers
1243 // the location of the first empty row.  (An empty row records null for its
1244 // receiver, and can be allocated for a newly-observed receiver type.)
1245 // Because there are two degrees of freedom in the state, a simple linear
1246 // search will not work; it must be a decision tree.  Hence this helper
1247 // function is recursive, to generate the required tree structured code.
1248 // It's the interpreter, so we are trading off code space for speed.
1249 // See below for example code.
1250 void InterpreterMacroAssembler::record_klass_in_profile_helper(
1251                                         Register receiver, Register mdp,
1252                                         Register reg2, int start_row,
1253                                         Label& done) {
1254   if (TypeProfileWidth == 0) {
1255     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1256   } else {
1257     record_item_in_profile_helper(receiver, mdp, reg2, 0, done, TypeProfileWidth,
1258         &VirtualCallData::receiver_offset, &VirtualCallData::receiver_count_offset);
1259   }
1260 }
1261 
1262 void InterpreterMacroAssembler::record_item_in_profile_helper(Register item, Register mdp,
1263                                         Register reg2, int start_row, Label& done, int total_rows,
1264                                         OffsetFunction item_offset_fn, OffsetFunction item_count_offset_fn) {
1265   int last_row = total_rows - 1;
1266   assert(start_row <= last_row, "must be work left to do");
1267   // Test this row for both the item and for null.
1268   // Take any of three different outcomes:
1269   //   1. found item => increment count and goto done
1270   //   2. found null => keep looking for case 1, maybe allocate this cell
1271   //   3. found something else => keep looking for cases 1 and 2
1272   // Case 3 is handled by a recursive call.
1273   for (int row = start_row; row <= last_row; row++) {
1274     Label next_test;
1275     bool test_for_null_also = (row == start_row);
1276 
1277     // See if the item is item[n].
1278     int item_offset = in_bytes(item_offset_fn(row));
1279     test_mdp_data_at(mdp, item_offset, item,
1280                      (test_for_null_also ? reg2 : noreg),
1281                      next_test);
1282     // (Reg2 now contains the item from the CallData.)
1283 
1284     // The item is item[n].  Increment count[n].
1285     int count_offset = in_bytes(item_count_offset_fn(row));
1286     increment_mdp_data_at(mdp, count_offset);
1287     b(done);
1288     bind(next_test);
1289 
1290     if (test_for_null_also) {
1291       Label found_null;
1292       // Failed the equality check on item[n]...  Test for null.
1293       if (start_row == last_row) {
1294         // The only thing left to do is handle the null case.
1295         cbz(reg2, found_null);
1296         // Item did not match any saved item and there is no empty row for it.
1297         // Increment total counter to indicate polymorphic case.
1298         increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1299         b(done);
1300         bind(found_null);
1301         break;
1302       }
1303       // Since null is rare, make it be the branch-taken case.
1304       cbz(reg2, found_null);
1305 
1306       // Put all the "Case 3" tests here.
1307       record_item_in_profile_helper(item, mdp, reg2, start_row + 1, done, total_rows,
1308         item_offset_fn, item_count_offset_fn);
1309 
1310       // Found a null.  Keep searching for a matching item,
1311       // but remember that this is an empty (unused) slot.
1312       bind(found_null);
1313     }
1314   }
1315 
1316   // In the fall-through case, we found no matching item, but we
1317   // observed the item[start_row] is null.
1318 
1319   // Fill in the item field and increment the count.
1320   int item_offset = in_bytes(item_offset_fn(start_row));
1321   set_mdp_data_at(mdp, item_offset, item);
1322   int count_offset = in_bytes(item_count_offset_fn(start_row));
1323   mov(reg2, DataLayout::counter_increment);
1324   set_mdp_data_at(mdp, count_offset, reg2);
1325   if (start_row > 0) {
1326     b(done);
1327   }
1328 }
1329 
1330 // Example state machine code for three profile rows:
1331 //   // main copy of decision tree, rooted at row[1]
1332 //   if (row[0].rec == rec) { row[0].incr(); goto done; }
1333 //   if (row[0].rec != nullptr) {
1334 //     // inner copy of decision tree, rooted at row[1]
1335 //     if (row[1].rec == rec) { row[1].incr(); goto done; }
1336 //     if (row[1].rec != nullptr) {
1337 //       // degenerate decision tree, rooted at row[2]
1338 //       if (row[2].rec == rec) { row[2].incr(); goto done; }
1339 //       if (row[2].rec != nullptr) { count.incr(); goto done; } // overflow
1340 //       row[2].init(rec); goto done;
1341 //     } else {
1342 //       // remember row[1] is empty
1343 //       if (row[2].rec == rec) { row[2].incr(); goto done; }
1344 //       row[1].init(rec); goto done;
1345 //     }
1346 //   } else {
1347 //     // remember row[0] is empty
1348 //     if (row[1].rec == rec) { row[1].incr(); goto done; }
1349 //     if (row[2].rec == rec) { row[2].incr(); goto done; }
1350 //     row[0].init(rec); goto done;
1351 //   }
1352 //   done:
1353 
1354 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
1355                                                         Register mdp, Register reg2) {
1356   assert(ProfileInterpreter, "must be profiling");
1357   Label done;
1358 
1359   record_klass_in_profile_helper(receiver, mdp, reg2, 0, done);
1360 
1361   bind (done);
1362 }
1363 
1364 void InterpreterMacroAssembler::profile_ret(Register return_bci,
1365                                             Register mdp) {
1366   if (ProfileInterpreter) {
1367     Label profile_continue;
1368     uint row;
1369 
1370     // If no method data exists, go to profile_continue.
1371     test_method_data_pointer(mdp, profile_continue);
1372 
1373     // Update the total ret count.
1374     increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1375 
1376     for (row = 0; row < RetData::row_limit(); row++) {
1377       Label next_test;
1378 
1379       // See if return_bci is equal to bci[n]:
1380       test_mdp_data_at(mdp,
1381                        in_bytes(RetData::bci_offset(row)),
1382                        return_bci, noreg,
1383                        next_test);
1384 
1385       // return_bci is equal to bci[n].  Increment the count.
1386       increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)));
1387 
1388       // The method data pointer needs to be updated to reflect the new target.
1389       update_mdp_by_offset(mdp,
1390                            in_bytes(RetData::bci_displacement_offset(row)));
1391       b(profile_continue);
1392       bind(next_test);
1393     }
1394 
1395     update_mdp_for_ret(return_bci);
1396 
1397     bind(profile_continue);
1398   }
1399 }
1400 
1401 void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
1402   if (ProfileInterpreter) {
1403     Label profile_continue;
1404 
1405     // If no method data exists, go to profile_continue.
1406     test_method_data_pointer(mdp, profile_continue);
1407 
1408     set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1409 
1410     // The method data pointer needs to be updated.
1411     int mdp_delta = in_bytes(BitData::bit_data_size());
1412     if (TypeProfileCasts) {
1413       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1414     }
1415     update_mdp_by_constant(mdp, mdp_delta);
1416 
1417     bind(profile_continue);
1418   }
1419 }
1420 
1421 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2) {
1422   if (ProfileInterpreter) {
1423     Label profile_continue;
1424 
1425     // If no method data exists, go to profile_continue.
1426     test_method_data_pointer(mdp, profile_continue);
1427 
1428     // The method data pointer needs to be updated.
1429     int mdp_delta = in_bytes(BitData::bit_data_size());
1430     if (TypeProfileCasts) {
1431       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1432 
1433       // Record the object type.
1434       record_klass_in_profile(klass, mdp, reg2);
1435     }
1436     update_mdp_by_constant(mdp, mdp_delta);
1437 
1438     bind(profile_continue);
1439   }
1440 }
1441 
1442 void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
1443   if (ProfileInterpreter) {
1444     Label profile_continue;
1445 
1446     // If no method data exists, go to profile_continue.
1447     test_method_data_pointer(mdp, profile_continue);
1448 
1449     // Update the default case count
1450     increment_mdp_data_at(mdp,
1451                           in_bytes(MultiBranchData::default_count_offset()));
1452 
1453     // The method data pointer needs to be updated.
1454     update_mdp_by_offset(mdp,
1455                          in_bytes(MultiBranchData::
1456                                   default_displacement_offset()));
1457 
1458     bind(profile_continue);
1459   }
1460 }
1461 
1462 void InterpreterMacroAssembler::profile_switch_case(Register index,
1463                                                     Register mdp,
1464                                                     Register reg2) {
1465   if (ProfileInterpreter) {
1466     Label profile_continue;
1467 
1468     // If no method data exists, go to profile_continue.
1469     test_method_data_pointer(mdp, profile_continue);
1470 
1471     // Build the base (index * per_case_size_in_bytes()) +
1472     // case_array_offset_in_bytes()
1473     movw(reg2, in_bytes(MultiBranchData::per_case_size()));
1474     movw(rscratch1, in_bytes(MultiBranchData::case_array_offset()));
1475     Assembler::maddw(index, index, reg2, rscratch1);
1476 
1477     // Update the case count
1478     increment_mdp_data_at(mdp,
1479                           index,
1480                           in_bytes(MultiBranchData::relative_count_offset()));
1481 
1482     // The method data pointer needs to be updated.
1483     update_mdp_by_offset(mdp,
1484                          index,
1485                          in_bytes(MultiBranchData::
1486                                   relative_displacement_offset()));
1487 
1488     bind(profile_continue);
1489   }
1490 }
1491 
1492 template <class ArrayData> void InterpreterMacroAssembler::profile_array_type(Register mdp,
1493                                                                               Register array,
1494                                                                               Register tmp) {
1495   if (ProfileInterpreter) {
1496     Label profile_continue;
1497 
1498     // If no method data exists, go to profile_continue.
1499     test_method_data_pointer(mdp, profile_continue);
1500 
1501     mov(tmp, array);
1502     profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset())));
1503 
1504     Label not_flat;
1505     test_non_flat_array_oop(array, tmp, not_flat);
1506 
1507     set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant());
1508 
1509     bind(not_flat);
1510 
1511     Label not_null_free;
1512     test_non_null_free_array_oop(array, tmp, not_null_free);
1513 
1514     set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant());
1515 
1516     bind(not_null_free);
1517 
1518     bind(profile_continue);
1519   }
1520 }
1521 
1522 template void InterpreterMacroAssembler::profile_array_type<ArrayLoadData>(Register mdp,
1523                                                                            Register array,
1524                                                                            Register tmp);
1525 template void InterpreterMacroAssembler::profile_array_type<ArrayStoreData>(Register mdp,
1526                                                                             Register array,
1527                                                                             Register tmp);
1528 
1529 void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) {
1530   if (ProfileInterpreter) {
1531     Label profile_continue;
1532 
1533     // If no method data exists, go to profile_continue.
1534     test_method_data_pointer(mdp, profile_continue);
1535 
1536     Label done, update;
1537     cbnz(element, update);
1538     set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1539     b(done);
1540 
1541     bind(update);
1542     load_klass(tmp, element);
1543 
1544     // Record the object type.
1545     record_klass_in_profile(tmp, mdp, tmp2);
1546 
1547     bind(done);
1548 
1549     // The method data pointer needs to be updated.
1550     update_mdp_by_constant(mdp, in_bytes(ArrayStoreData::array_store_data_size()));
1551 
1552     bind(profile_continue);
1553   }
1554 }
1555 
1556 
1557 void InterpreterMacroAssembler::profile_element_type(Register mdp,
1558                                                      Register element,
1559                                                      Register tmp) {
1560   if (ProfileInterpreter) {
1561     Label profile_continue;
1562 
1563     // If no method data exists, go to profile_continue.
1564     test_method_data_pointer(mdp, profile_continue);
1565 
1566     mov(tmp, element);
1567     profile_obj_type(tmp, Address(mdp, in_bytes(ArrayLoadData::element_offset())));
1568 
1569     // The method data pointer needs to be updated.
1570     update_mdp_by_constant(mdp, in_bytes(ArrayLoadData::array_load_data_size()));
1571 
1572     bind(profile_continue);
1573   }
1574 }
1575 
1576 void InterpreterMacroAssembler::profile_acmp(Register mdp,
1577                                              Register left,
1578                                              Register right,
1579                                              Register tmp) {
1580   if (ProfileInterpreter) {
1581     Label profile_continue;
1582 
1583     // If no method data exists, go to profile_continue.
1584     test_method_data_pointer(mdp, profile_continue);
1585 
1586     mov(tmp, left);
1587     profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::left_offset())));
1588 
1589     Label left_not_inline_type;
1590     test_oop_is_not_inline_type(left, tmp, left_not_inline_type);
1591     set_mdp_flag_at(mdp, ACmpData::left_inline_type_byte_constant());
1592     bind(left_not_inline_type);
1593 
1594     mov(tmp, right);
1595     profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::right_offset())));
1596 
1597     Label right_not_inline_type;
1598     test_oop_is_not_inline_type(right, tmp, right_not_inline_type);
1599     set_mdp_flag_at(mdp, ACmpData::right_inline_type_byte_constant());
1600     bind(right_not_inline_type);
1601 
1602     bind(profile_continue);
1603   }
1604 }
1605 
1606 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1607   if (state == atos) {
1608     MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1609   }
1610 }
1611 
1612 void InterpreterMacroAssembler::notify_method_entry() {
1613   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1614   // track stack depth.  If it is possible to enter interp_only_mode we add
1615   // the code to check if the event should be sent.
1616   if (JvmtiExport::can_post_interpreter_events()) {
1617     Label L;
1618     ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1619     cbzw(r3, L);
1620     call_VM(noreg, CAST_FROM_FN_PTR(address,
1621                                     InterpreterRuntime::post_method_entry));
1622     bind(L);
1623   }
1624 
1625   if (DTraceMethodProbes) {
1626     get_method(c_rarg1);
1627     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
1628                  rthread, c_rarg1);
1629   }
1630 
1631   // RedefineClasses() tracing support for obsolete method entry
1632   if (log_is_enabled(Trace, redefine, class, obsolete)) {
1633     get_method(c_rarg1);
1634     call_VM_leaf(
1635       CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
1636       rthread, c_rarg1);
1637   }
1638 
1639  }
1640 
1641 
1642 void InterpreterMacroAssembler::notify_method_exit(
1643     TosState state, NotifyMethodExitMode mode) {
1644   // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1645   // track stack depth.  If it is possible to enter interp_only_mode we add
1646   // the code to check if the event should be sent.
1647   if (mode == NotifyJVMTI && JvmtiExport::can_post_interpreter_events()) {
1648     Label L;
1649     // Note: frame::interpreter_frame_result has a dependency on how the
1650     // method result is saved across the call to post_method_exit. If this
1651     // is changed then the interpreter_frame_result implementation will
1652     // need to be updated too.
1653 
1654     // template interpreter will leave the result on the top of the stack.
1655     push(state);
1656     ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1657     cbz(r3, L);
1658     call_VM(noreg,
1659             CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
1660     bind(L);
1661     pop(state);
1662   }
1663 
1664   if (DTraceMethodProbes) {
1665     push(state);
1666     get_method(c_rarg1);
1667     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
1668                  rthread, c_rarg1);
1669     pop(state);
1670   }
1671 }
1672 
1673 
1674 // Jump if ((*counter_addr += increment) & mask) satisfies the condition.
1675 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
1676                                                         int increment, Address mask,
1677                                                         Register scratch, Register scratch2,
1678                                                         bool preloaded, Condition cond,
1679                                                         Label* where) {
1680   if (!preloaded) {
1681     ldrw(scratch, counter_addr);
1682   }
1683   add(scratch, scratch, increment);
1684   strw(scratch, counter_addr);
1685   ldrw(scratch2, mask);
1686   ands(scratch, scratch, scratch2);
1687   br(cond, *where);
1688 }
1689 
1690 void InterpreterMacroAssembler::call_VM_leaf_base(address entry_point,
1691                                                   int number_of_arguments) {
1692   // interpreter specific
1693   //
1694   // Note: No need to save/restore rbcp & rlocals pointer since these
1695   //       are callee saved registers and no blocking/ GC can happen
1696   //       in leaf calls.
1697 #ifdef ASSERT
1698   {
1699     Label L;
1700     ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1701     cbz(rscratch1, L);
1702     stop("InterpreterMacroAssembler::call_VM_leaf_base:"
1703          " last_sp != nullptr");
1704     bind(L);
1705   }
1706 #endif /* ASSERT */
1707   // super call
1708   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
1709 }
1710 
1711 void InterpreterMacroAssembler::call_VM_base(Register oop_result,
1712                                              Register java_thread,
1713                                              Register last_java_sp,
1714                                              address  entry_point,
1715                                              int      number_of_arguments,
1716                                              bool     check_exceptions) {
1717   // interpreter specific
1718   //
1719   // Note: Could avoid restoring locals ptr (callee saved) - however doesn't
1720   //       really make a difference for these runtime calls, since they are
1721   //       slow anyway. Btw., bcp must be saved/restored since it may change
1722   //       due to GC.
1723   // assert(java_thread == noreg , "not expecting a precomputed java thread");
1724   save_bcp();
1725 #ifdef ASSERT
1726   {
1727     Label L;
1728     ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1729     cbz(rscratch1, L);
1730     stop("InterpreterMacroAssembler::call_VM_base:"
1731          " last_sp != nullptr");
1732     bind(L);
1733   }
1734 #endif /* ASSERT */
1735   // super call
1736   MacroAssembler::call_VM_base(oop_result, noreg, last_java_sp,
1737                                entry_point, number_of_arguments,
1738                      check_exceptions);
1739 // interpreter specific
1740   restore_bcp();
1741   restore_locals();
1742 }
1743 
1744 void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result,
1745                                                     address entry_point,
1746                                                     Register arg_1) {
1747   assert(arg_1 == c_rarg1, "");
1748   Label resume_pc, not_preempted;
1749 
1750 #ifdef ASSERT
1751   {
1752     Label L;
1753     ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1754     cbz(rscratch1, L);
1755     stop("Should not have alternate return address set");
1756     bind(L);
1757   }
1758 #endif /* ASSERT */
1759 
1760   // Force freeze slow path.
1761   push_cont_fastpath();
1762 
1763   // Make VM call. In case of preemption set last_pc to the one we want to resume to.
1764   adr(rscratch1, resume_pc);
1765   str(rscratch1, Address(rthread, JavaThread::last_Java_pc_offset()));
1766   call_VM_base(oop_result, noreg, noreg, entry_point, 1, false /*check_exceptions*/);
1767 
1768   pop_cont_fastpath();
1769 
1770   // Check if preempted.
1771   ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1772   cbz(rscratch1, not_preempted);
1773   str(zr, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1774   br(rscratch1);
1775 
1776   // In case of preemption, this is where we will resume once we finally acquire the monitor.
1777   bind(resume_pc);
1778   restore_after_resume(false /* is_native */);
1779 
1780   bind(not_preempted);
1781 }
1782 
1783 void InterpreterMacroAssembler::restore_after_resume(bool is_native) {
1784   lea(rscratch1, ExternalAddress(Interpreter::cont_resume_interpreter_adapter()));
1785   blr(rscratch1);
1786   if (is_native) {
1787     // On resume we need to set up stack as expected
1788     push(dtos);
1789     push(ltos);
1790   }
1791 }
1792 
1793 void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {
1794   assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
1795   Label update, next, none;
1796 
1797   verify_oop(obj);
1798 
1799   cbnz(obj, update);
1800   orptr(mdo_addr, TypeEntries::null_seen);
1801   b(next);
1802 
1803   bind(update);
1804   load_klass(obj, obj);
1805 
1806   ldr(rscratch1, mdo_addr);
1807   eor(obj, obj, rscratch1);
1808   tst(obj, TypeEntries::type_klass_mask);
1809   br(Assembler::EQ, next); // klass seen before, nothing to
1810                            // do. The unknown bit may have been
1811                            // set already but no need to check.
1812 
1813   tbnz(obj, exact_log2(TypeEntries::type_unknown), next);
1814   // already unknown. Nothing to do anymore.
1815 
1816   cbz(rscratch1, none);
1817   cmp(rscratch1, (u1)TypeEntries::null_seen);
1818   br(Assembler::EQ, none);
1819   // There is a chance that the checks above
1820   // fail if another thread has just set the
1821   // profiling to this obj's klass
1822   eor(obj, obj, rscratch1); // get back original value before XOR
1823   ldr(rscratch1, mdo_addr);
1824   eor(obj, obj, rscratch1);
1825   tst(obj, TypeEntries::type_klass_mask);
1826   br(Assembler::EQ, next);
1827 
1828   // different than before. Cannot keep accurate profile.
1829   orptr(mdo_addr, TypeEntries::type_unknown);
1830   b(next);
1831 
1832   bind(none);
1833   // first time here. Set profile type.
1834   str(obj, mdo_addr);
1835 #ifdef ASSERT
1836   andr(obj, obj, TypeEntries::type_mask);
1837   verify_klass_ptr(obj);
1838 #endif
1839 
1840   bind(next);
1841 }
1842 
1843 void InterpreterMacroAssembler::profile_arguments_type(Register mdp, Register callee, Register tmp, bool is_virtual) {
1844   if (!ProfileInterpreter) {
1845     return;
1846   }
1847 
1848   if (MethodData::profile_arguments() || MethodData::profile_return()) {
1849     Label profile_continue;
1850 
1851     test_method_data_pointer(mdp, profile_continue);
1852 
1853     int off_to_start = is_virtual ? in_bytes(VirtualCallData::virtual_call_data_size()) : in_bytes(CounterData::counter_data_size());
1854 
1855     ldrb(rscratch1, Address(mdp, in_bytes(DataLayout::tag_offset()) - off_to_start));
1856     cmp(rscratch1, u1(is_virtual ? DataLayout::virtual_call_type_data_tag : DataLayout::call_type_data_tag));
1857     br(Assembler::NE, profile_continue);
1858 
1859     if (MethodData::profile_arguments()) {
1860       Label done;
1861       int off_to_args = in_bytes(TypeEntriesAtCall::args_data_offset());
1862 
1863       for (int i = 0; i < TypeProfileArgsLimit; i++) {
1864         if (i > 0 || MethodData::profile_return()) {
1865           // If return value type is profiled we may have no argument to profile
1866           ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1867           sub(tmp, tmp, i*TypeStackSlotEntries::per_arg_count());
1868           cmp(tmp, (u1)TypeStackSlotEntries::per_arg_count());
1869           add(rscratch1, mdp, off_to_args);
1870           br(Assembler::LT, done);
1871         }
1872         ldr(tmp, Address(callee, Method::const_offset()));
1873         load_unsigned_short(tmp, Address(tmp, ConstMethod::size_of_parameters_offset()));
1874         // stack offset o (zero based) from the start of the argument
1875         // list, for n arguments translates into offset n - o - 1 from
1876         // the end of the argument list
1877         ldr(rscratch1, Address(mdp, in_bytes(TypeEntriesAtCall::stack_slot_offset(i))));
1878         sub(tmp, tmp, rscratch1);
1879         sub(tmp, tmp, 1);
1880         Address arg_addr = argument_address(tmp);
1881         ldr(tmp, arg_addr);
1882 
1883         Address mdo_arg_addr(mdp, in_bytes(TypeEntriesAtCall::argument_type_offset(i)));
1884         profile_obj_type(tmp, mdo_arg_addr);
1885 
1886         int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
1887         off_to_args += to_add;
1888       }
1889 
1890       if (MethodData::profile_return()) {
1891         ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1892         sub(tmp, tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
1893       }
1894 
1895       add(rscratch1, mdp, off_to_args);
1896       bind(done);
1897       mov(mdp, rscratch1);
1898 
1899       if (MethodData::profile_return()) {
1900         // We're right after the type profile for the last
1901         // argument. tmp is the number of cells left in the
1902         // CallTypeData/VirtualCallTypeData to reach its end. Non null
1903         // if there's a return to profile.
1904         assert(SingleTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
1905         add(mdp, mdp, tmp, LSL, exact_log2(DataLayout::cell_size));
1906       }
1907       str(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1908     } else {
1909       assert(MethodData::profile_return(), "either profile call args or call ret");
1910       update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
1911     }
1912 
1913     // mdp points right after the end of the
1914     // CallTypeData/VirtualCallTypeData, right after the cells for the
1915     // return value type if there's one
1916 
1917     bind(profile_continue);
1918   }
1919 }
1920 
1921 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
1922   assert_different_registers(mdp, ret, tmp, rbcp);
1923   if (ProfileInterpreter && MethodData::profile_return()) {
1924     Label profile_continue, done;
1925 
1926     test_method_data_pointer(mdp, profile_continue);
1927 
1928     if (MethodData::profile_return_jsr292_only()) {
1929       assert(Method::intrinsic_id_size_in_bytes() == 2, "assuming Method::_intrinsic_id is u2");
1930 
1931       // If we don't profile all invoke bytecodes we must make sure
1932       // it's a bytecode we indeed profile. We can't go back to the
1933       // beginning of the ProfileData we intend to update to check its
1934       // type because we're right after it and we don't known its
1935       // length
1936       Label do_profile;
1937       ldrb(rscratch1, Address(rbcp, 0));
1938       cmp(rscratch1, (u1)Bytecodes::_invokedynamic);
1939       br(Assembler::EQ, do_profile);
1940       cmp(rscratch1, (u1)Bytecodes::_invokehandle);
1941       br(Assembler::EQ, do_profile);
1942       get_method(tmp);
1943       ldrh(rscratch1, Address(tmp, Method::intrinsic_id_offset()));
1944       subs(zr, rscratch1, static_cast<int>(vmIntrinsics::_compiledLambdaForm));
1945       br(Assembler::NE, profile_continue);
1946 
1947       bind(do_profile);
1948     }
1949 
1950     Address mdo_ret_addr(mdp, -in_bytes(SingleTypeEntry::size()));
1951     mov(tmp, ret);
1952     profile_obj_type(tmp, mdo_ret_addr);
1953 
1954     bind(profile_continue);
1955   }
1956 }
1957 
1958 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
1959   assert_different_registers(rscratch1, rscratch2, mdp, tmp1, tmp2);
1960   if (ProfileInterpreter && MethodData::profile_parameters()) {
1961     Label profile_continue, done;
1962 
1963     test_method_data_pointer(mdp, profile_continue);
1964 
1965     // Load the offset of the area within the MDO used for
1966     // parameters. If it's negative we're not profiling any parameters
1967     ldrw(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
1968     tbnz(tmp1, 31, profile_continue);  // i.e. sign bit set
1969 
1970     // Compute a pointer to the area for parameters from the offset
1971     // and move the pointer to the slot for the last
1972     // parameters. Collect profiling from last parameter down.
1973     // mdo start + parameters offset + array length - 1
1974     add(mdp, mdp, tmp1);
1975     ldr(tmp1, Address(mdp, ArrayData::array_len_offset()));
1976     sub(tmp1, tmp1, TypeStackSlotEntries::per_arg_count());
1977 
1978     Label loop;
1979     bind(loop);
1980 
1981     int off_base = in_bytes(ParametersTypeData::stack_slot_offset(0));
1982     int type_base = in_bytes(ParametersTypeData::type_offset(0));
1983     int per_arg_scale = exact_log2(DataLayout::cell_size);
1984     add(rscratch1, mdp, off_base);
1985     add(rscratch2, mdp, type_base);
1986 
1987     Address arg_off(rscratch1, tmp1, Address::lsl(per_arg_scale));
1988     Address arg_type(rscratch2, tmp1, Address::lsl(per_arg_scale));
1989 
1990     // load offset on the stack from the slot for this parameter
1991     ldr(tmp2, arg_off);
1992     neg(tmp2, tmp2);
1993     // read the parameter from the local area
1994     ldr(tmp2, Address(rlocals, tmp2, Address::lsl(Interpreter::logStackElementSize)));
1995 
1996     // profile the parameter
1997     profile_obj_type(tmp2, arg_type);
1998 
1999     // go to next parameter
2000     subs(tmp1, tmp1, TypeStackSlotEntries::per_arg_count());
2001     br(Assembler::GE, loop);
2002 
2003     bind(profile_continue);
2004   }
2005 }
2006 
2007 void InterpreterMacroAssembler::load_resolved_indy_entry(Register cache, Register index) {
2008   // Get index out of bytecode pointer, get_cache_entry_pointer_at_bcp
2009   get_cache_index_at_bcp(index, 1, sizeof(u4));
2010   // Get address of invokedynamic array
2011   ldr(cache, Address(rcpool, in_bytes(ConstantPoolCache::invokedynamic_entries_offset())));
2012   // Scale the index to be the entry index * sizeof(ResolvedIndyEntry)
2013   lsl(index, index, log2i_exact(sizeof(ResolvedIndyEntry)));
2014   add(cache, cache, Array<ResolvedIndyEntry>::base_offset_in_bytes());
2015   lea(cache, Address(cache, index));
2016 }
2017 
2018 void InterpreterMacroAssembler::load_field_entry(Register cache, Register index, int bcp_offset) {
2019   // Get index out of bytecode pointer
2020   get_cache_index_at_bcp(index, bcp_offset, sizeof(u2));
2021   // Take shortcut if the size is a power of 2
2022   if (is_power_of_2(sizeof(ResolvedFieldEntry))) {
2023     lsl(index, index, log2i_exact(sizeof(ResolvedFieldEntry))); // Scale index by power of 2
2024   } else {
2025     mov(cache, sizeof(ResolvedFieldEntry));
2026     mul(index, index, cache); // Scale the index to be the entry index * sizeof(ResolvedFieldEntry)
2027   }
2028   // Get address of field entries array
2029   ldr(cache, Address(rcpool, ConstantPoolCache::field_entries_offset()));
2030   add(cache, cache, Array<ResolvedFieldEntry>::base_offset_in_bytes());
2031   lea(cache, Address(cache, index));
2032   // Prevents stale data from being read after the bytecode is patched to the fast bytecode
2033   membar(MacroAssembler::LoadLoad);
2034 }
2035 
2036 void InterpreterMacroAssembler::load_method_entry(Register cache, Register index, int bcp_offset) {
2037   // Get index out of bytecode pointer
2038   get_cache_index_at_bcp(index, bcp_offset, sizeof(u2));
2039   mov(cache, sizeof(ResolvedMethodEntry));
2040   mul(index, index, cache); // Scale the index to be the entry index * sizeof(ResolvedMethodEntry)
2041 
2042   // Get address of field entries array
2043   ldr(cache, Address(rcpool, ConstantPoolCache::method_entries_offset()));
2044   add(cache, cache, Array<ResolvedMethodEntry>::base_offset_in_bytes());
2045   lea(cache, Address(cache, index));
2046 }