1 /*
   2  * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.inline.hpp"
  29 #include "gc/shared/barrierSetAssembler.hpp"
  30 #include "interpreter/bytecodeHistogram.hpp"
  31 #include "interpreter/bytecodeTracer.hpp"
  32 #include "interpreter/interp_masm.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "interpreter/interpreterRuntime.hpp"
  35 #include "interpreter/templateInterpreterGenerator.hpp"
  36 #include "interpreter/templateTable.hpp"
  37 #include "memory/resourceArea.hpp"
  38 #include "oops/arrayOop.hpp"
  39 #include "oops/method.hpp"
  40 #include "oops/methodData.hpp"
  41 #include "oops/oop.inline.hpp"
  42 #include "prims/jvmtiExport.hpp"
  43 #include "prims/jvmtiThreadState.hpp"
  44 #include "runtime/arguments.hpp"
  45 #include "runtime/deoptimization.hpp"
  46 #include "runtime/frame.inline.hpp"
  47 #include "runtime/jniHandles.hpp"
  48 #include "runtime/sharedRuntime.hpp"
  49 #include "runtime/stubRoutines.hpp"
  50 #include "runtime/synchronizer.hpp"
  51 #include "runtime/timer.hpp"
  52 #include "runtime/vframeArray.hpp"
  53 #include "utilities/debug.hpp"
  54 #include "utilities/powerOfTwo.hpp"
  55 #include <sys/types.h>
  56 
  57 #ifndef PRODUCT
  58 #include "oops/method.hpp"
  59 #endif // !PRODUCT
  60 
  61 // Size of interpreter code.  Increase if too small.  Interpreter will
  62 // fail with a guarantee ("not enough space for interpreter generation");
  63 // if too small.
  64 // Run with +PrintInterpreter to get the VM to print out the size.
  65 // Max size with JVMTI
  66 int TemplateInterpreter::InterpreterCodeSize = 256 * 1024;
  67 
  68 #define __ _masm->
  69 
  70 //-----------------------------------------------------------------------------
  71 
  72 address TemplateInterpreterGenerator::generate_slow_signature_handler() {
  73   address entry = __ pc();
  74 
  75   __ andi(esp, esp, -16);
  76   __ mv(c_rarg3, esp);
  77   // xmethod
  78   // xlocals
  79   // c_rarg3: first stack arg - wordSize
  80   // adjust sp
  81 
  82   __ addi(sp, c_rarg3, -18 * wordSize);
  83   __ addi(sp, sp, -2 * wordSize);
  84   __ sd(lr, Address(sp, 0));
  85 
  86   __ call_VM(noreg,
  87              CAST_FROM_FN_PTR(address,
  88                               InterpreterRuntime::slow_signature_handler),
  89              xmethod, xlocals, c_rarg3);
  90 
  91   // x10: result handler
  92 
  93   // Stack layout:
  94   // sp: return address           <- sp
  95   //      1 garbage
  96   //      8 integer args (if static first is unused)
  97   //      1 float/double identifiers
  98   //      8 double args
  99   //        stack args              <- esp
 100   //        garbage
 101   //        expression stack bottom
 102   //        bcp (NULL)
 103   //        ...
 104 
 105   // Restore LR
 106   __ ld(lr, Address(sp, 0));
 107   __ addi(sp, sp , 2 * wordSize);
 108 
 109   // Do FP first so we can use c_rarg3 as temp
 110   __ lwu(c_rarg3, Address(sp, 9 * wordSize)); // float/double identifiers
 111 
 112   for (int i = 0; i < Argument::n_float_register_parameters_c; i++) {
 113     const FloatRegister r = g_FPArgReg[i];
 114     Label d, done;
 115 
 116     __ andi(t0, c_rarg3, 1UL << i);
 117     __ bnez(t0, d);
 118     __ flw(r, Address(sp, (10 + i) * wordSize));
 119     __ j(done);
 120     __ bind(d);
 121     __ fld(r, Address(sp, (10 + i) * wordSize));
 122     __ bind(done);
 123   }
 124 
 125   // c_rarg0 contains the result from the call of
 126   // InterpreterRuntime::slow_signature_handler so we don't touch it
 127   // here.  It will be loaded with the JNIEnv* later.
 128   for (int i = 1; i < Argument::n_int_register_parameters_c; i++) {
 129     const Register rm = g_INTArgReg[i];
 130     __ ld(rm, Address(sp, i * wordSize));
 131   }
 132 
 133   __ addi(sp, sp, 18 * wordSize);
 134   __ ret();
 135 
 136   return entry;
 137 }
 138 
 139 // Various method entries
 140 address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) {
 141   // xmethod: Method*
 142   // x30: sender sp
 143   // esp: args
 144 
 145   if (!InlineIntrinsics) {
 146     return NULL; // Generate a vanilla entry
 147   }
 148 
 149   // These don't need a safepoint check because they aren't virtually
 150   // callable. We won't enter these intrinsics from compiled code.
 151   // If in the future we added an intrinsic which was virtually callable
 152   // we'd have to worry about how to safepoint so that this code is used.
 153 
 154   // mathematical functions inlined by compiler
 155   // (interpreter must provide identical implementation
 156   // in order to avoid monotonicity bugs when switching
 157   // from interpreter to compiler in the middle of some
 158   // computation)
 159   //
 160   // stack:
 161   //        [ arg ] <-- esp
 162   //        [ arg ]
 163   // retaddr in lr
 164 
 165   address fn = NULL;
 166   address entry_point = NULL;
 167   Register continuation = lr;
 168   switch (kind) {
 169     case Interpreter::java_lang_math_abs:
 170       entry_point = __ pc();
 171       __ fld(f10, Address(esp));
 172       __ fabs_d(f10, f10);
 173       __ mv(sp, x30); // Restore caller's SP
 174       break;
 175     case Interpreter::java_lang_math_sqrt:
 176       entry_point = __ pc();
 177       __ fld(f10, Address(esp));
 178       __ fsqrt_d(f10, f10);
 179       __ mv(sp, x30);
 180       break;
 181     case Interpreter::java_lang_math_sin :
 182       entry_point = __ pc();
 183       __ fld(f10, Address(esp));
 184       __ mv(sp, x30);
 185       __ mv(x9, lr);
 186       continuation = x9;  // The first callee-saved register
 187       if (StubRoutines::dsin() == NULL) {
 188         fn = CAST_FROM_FN_PTR(address, SharedRuntime::dsin);
 189       } else {
 190         fn = CAST_FROM_FN_PTR(address, StubRoutines::dsin());
 191       }
 192       __ mv(t0, fn);
 193       __ jalr(t0);
 194       break;
 195     case Interpreter::java_lang_math_cos :
 196       entry_point = __ pc();
 197       __ fld(f10, Address(esp));
 198       __ mv(sp, x30);
 199       __ mv(x9, lr);
 200       continuation = x9;  // The first callee-saved register
 201       if (StubRoutines::dcos() == NULL) {
 202         fn = CAST_FROM_FN_PTR(address, SharedRuntime::dcos);
 203       } else {
 204         fn = CAST_FROM_FN_PTR(address, StubRoutines::dcos());
 205       }
 206       __ mv(t0, fn);
 207       __ jalr(t0);
 208       break;
 209     case Interpreter::java_lang_math_tan :
 210       entry_point = __ pc();
 211       __ fld(f10, Address(esp));
 212       __ mv(sp, x30);
 213       __ mv(x9, lr);
 214       continuation = x9;  // The first callee-saved register
 215       if (StubRoutines::dtan() == NULL) {
 216         fn = CAST_FROM_FN_PTR(address, SharedRuntime::dtan);
 217       } else {
 218         fn = CAST_FROM_FN_PTR(address, StubRoutines::dtan());
 219       }
 220       __ mv(t0, fn);
 221       __ jalr(t0);
 222       break;
 223     case Interpreter::java_lang_math_log :
 224       entry_point = __ pc();
 225       __ fld(f10, Address(esp));
 226       __ mv(sp, x30);
 227       __ mv(x9, lr);
 228       continuation = x9;  // The first callee-saved register
 229       if (StubRoutines::dlog() == NULL) {
 230         fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog);
 231       } else {
 232         fn = CAST_FROM_FN_PTR(address, StubRoutines::dlog());
 233       }
 234       __ mv(t0, fn);
 235       __ jalr(t0);
 236       break;
 237     case Interpreter::java_lang_math_log10 :
 238       entry_point = __ pc();
 239       __ fld(f10, Address(esp));
 240       __ mv(sp, x30);
 241       __ mv(x9, lr);
 242       continuation = x9;  // The first callee-saved register
 243       if (StubRoutines::dlog10() == NULL) {
 244         fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog10);
 245       } else {
 246         fn = CAST_FROM_FN_PTR(address, StubRoutines::dlog10());
 247       }
 248       __ mv(t0, fn);
 249       __ jalr(t0);
 250       break;
 251     case Interpreter::java_lang_math_exp :
 252       entry_point = __ pc();
 253       __ fld(f10, Address(esp));
 254       __ mv(sp, x30);
 255       __ mv(x9, lr);
 256       continuation = x9;  // The first callee-saved register
 257       if (StubRoutines::dexp() == NULL) {
 258         fn = CAST_FROM_FN_PTR(address, SharedRuntime::dexp);
 259       } else {
 260         fn = CAST_FROM_FN_PTR(address, StubRoutines::dexp());
 261       }
 262       __ mv(t0, fn);
 263       __ jalr(t0);
 264       break;
 265     case Interpreter::java_lang_math_pow :
 266       entry_point = __ pc();
 267       __ mv(x9, lr);
 268       continuation = x9;
 269       __ fld(f10, Address(esp, 2 * Interpreter::stackElementSize));
 270       __ fld(f11, Address(esp));
 271       __ mv(sp, x30);
 272       if (StubRoutines::dpow() == NULL) {
 273         fn = CAST_FROM_FN_PTR(address, SharedRuntime::dpow);
 274       } else {
 275         fn = CAST_FROM_FN_PTR(address, StubRoutines::dpow());
 276       }
 277       __ mv(t0, fn);
 278       __ jalr(t0);
 279       break;
 280     case Interpreter::java_lang_math_fmaD :
 281       if (UseFMA) {
 282         entry_point = __ pc();
 283         __ fld(f10, Address(esp, 4 * Interpreter::stackElementSize));
 284         __ fld(f11, Address(esp, 2 * Interpreter::stackElementSize));
 285         __ fld(f12, Address(esp));
 286         __ fmadd_d(f10, f10, f11, f12);
 287         __ mv(sp, x30); // Restore caller's SP
 288       }
 289       break;
 290     case Interpreter::java_lang_math_fmaF :
 291       if (UseFMA) {
 292         entry_point = __ pc();
 293         __ flw(f10, Address(esp, 2 * Interpreter::stackElementSize));
 294         __ flw(f11, Address(esp, Interpreter::stackElementSize));
 295         __ flw(f12, Address(esp));
 296         __ fmadd_s(f10, f10, f11, f12);
 297         __ mv(sp, x30); // Restore caller's SP
 298       }
 299       break;
 300     default:
 301       ;
 302   }
 303   if (entry_point != NULL) {
 304     __ jr(continuation);
 305   }
 306 
 307   return entry_point;
 308 }
 309 
 310 // Abstract method entry
 311 // Attempt to execute abstract method. Throw exception
 312 address TemplateInterpreterGenerator::generate_abstract_entry(void) {
 313   // xmethod: Method*
 314   // x30: sender SP
 315 
 316   address entry_point = __ pc();
 317 
 318   // abstract method entry
 319 
 320   //  pop return address, reset last_sp to NULL
 321   __ empty_expression_stack();
 322   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
 323   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
 324 
 325   // throw exception
 326   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
 327                                      InterpreterRuntime::throw_AbstractMethodErrorWithMethod),
 328                                      xmethod);
 329   // the call_VM checks for exception, so we should never return here.
 330   __ should_not_reach_here();
 331 
 332   return entry_point;
 333 }
 334 
 335 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
 336   address entry = __ pc();
 337 
 338 #ifdef ASSERT
 339   {
 340     Label L;
 341     __ ld(t0, Address(fp, frame::interpreter_frame_monitor_block_top_offset * wordSize));
 342     __ mv(t1, sp);
 343     // maximal sp for current fp (stack grows negative)
 344     // check if frame is complete
 345     __ bge(t0, t1, L);
 346     __ stop ("interpreter frame not set up");
 347     __ bind(L);
 348   }
 349 #endif // ASSERT
 350   // Restore bcp under the assumption that the current frame is still
 351   // interpreted
 352   __ restore_bcp();
 353 
 354   // expression stack must be empty before entering the VM if an
 355   // exception happened
 356   __ empty_expression_stack();
 357   // throw exception
 358   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_StackOverflowError));
 359   return entry;
 360 }
 361 
 362 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler() {
 363   address entry = __ pc();
 364   // expression stack must be empty before entering the VM if an
 365   // exception happened
 366   __ empty_expression_stack();
 367   // setup parameters
 368 
 369   // convention: expect aberrant index in register x11
 370   __ zero_ext(c_rarg2, x11, 32);
 371   // convention: expect array in register x13
 372   __ mv(c_rarg1, x13);
 373   __ call_VM(noreg,
 374              CAST_FROM_FN_PTR(address,
 375                               InterpreterRuntime::
 376                               throw_ArrayIndexOutOfBoundsException),
 377              c_rarg1, c_rarg2);
 378   return entry;
 379 }
 380 
 381 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
 382   address entry = __ pc();
 383 
 384   // object is at TOS
 385   __ pop_reg(c_rarg1);
 386 
 387   // expression stack must be empty before entering the VM if an
 388   // exception happened
 389   __ empty_expression_stack();
 390 
 391   __ call_VM(noreg,
 392              CAST_FROM_FN_PTR(address,
 393                               InterpreterRuntime::
 394                               throw_ClassCastException),
 395              c_rarg1);
 396   return entry;
 397 }
 398 
 399 address TemplateInterpreterGenerator::generate_exception_handler_common(
 400   const char* name, const char* message, bool pass_oop) {
 401   assert(!pass_oop || message == NULL, "either oop or message but not both");
 402   address entry = __ pc();
 403   if (pass_oop) {
 404     // object is at TOS
 405     __ pop_reg(c_rarg2);
 406   }
 407   // expression stack must be empty before entering the VM if an
 408   // exception happened
 409   __ empty_expression_stack();
 410   // setup parameters
 411   __ la(c_rarg1, Address((address)name));
 412   if (pass_oop) {
 413     __ call_VM(x10, CAST_FROM_FN_PTR(address,
 414                                      InterpreterRuntime::
 415                                      create_klass_exception),
 416                c_rarg1, c_rarg2);
 417   } else {
 418     // kind of lame ExternalAddress can't take NULL because
 419     // external_word_Relocation will assert.
 420     if (message != NULL) {
 421       __ la(c_rarg2, Address((address)message));
 422     } else {
 423       __ mv(c_rarg2, NULL_WORD);
 424     }
 425     __ call_VM(x10,
 426                CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception),
 427                c_rarg1, c_rarg2);
 428   }
 429   // throw exception
 430   __ j(address(Interpreter::throw_exception_entry()));
 431   return entry;
 432 }
 433 
 434 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {
 435   address entry = __ pc();
 436 
 437   // Restore stack bottom in case i2c adjusted stack
 438   __ ld(esp, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize));
 439   // and NULL it as marker that esp is now tos until next java call
 440   __ sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize));
 441   __ restore_bcp();
 442   __ restore_locals();
 443   __ restore_constant_pool_cache();
 444   __ get_method(xmethod);
 445 
 446   if (state == atos) {
 447     Register obj = x10;
 448     Register mdp = x11;
 449     Register tmp = x12;
 450     __ ld(mdp, Address(xmethod, Method::method_data_offset()));
 451     __ profile_return_type(mdp, obj, tmp);
 452   }
 453 
 454   // Pop N words from the stack
 455   __ get_cache_and_index_at_bcp(x11, x12, 1, index_size);
 456   __ ld(x11, Address(x11, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));
 457   __ andi(x11, x11, ConstantPoolCacheEntry::parameter_size_mask);
 458 
 459   __ slli(t0, x11, 3);
 460   __ add(esp, esp, t0);
 461 
 462   // Restore machine SP
 463   __ ld(t0, Address(xmethod, Method::const_offset()));
 464   __ lhu(t0, Address(t0, ConstMethod::max_stack_offset()));
 465   __ addi(t0, t0, frame::interpreter_frame_monitor_size() + 2);
 466   __ ld(t1,
 467         Address(fp, frame::interpreter_frame_initial_sp_offset * wordSize));
 468   __ slli(t0, t0, 3);
 469   __ sub(t0, t1, t0);
 470   __ andi(sp, t0, -16);
 471 
 472  __ check_and_handle_popframe(xthread);
 473  __ check_and_handle_earlyret(xthread);
 474 
 475   __ get_dispatch();
 476   __ dispatch_next(state, step);
 477 
 478   return entry;
 479 }
 480 
 481 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state,
 482                                                                int step,
 483                                                                address continuation) {
 484   address entry = __ pc();
 485   __ restore_bcp();
 486   __ restore_locals();
 487   __ restore_constant_pool_cache();
 488   __ get_method(xmethod);
 489   __ get_dispatch();
 490 
 491   // Calculate stack limit
 492   __ ld(t0, Address(xmethod, Method::const_offset()));
 493   __ lhu(t0, Address(t0, ConstMethod::max_stack_offset()));
 494   __ addi(t0, t0, frame::interpreter_frame_monitor_size() + 2);
 495   __ ld(t1, Address(fp, frame::interpreter_frame_initial_sp_offset * wordSize));
 496   __ slli(t0, t0, 3);
 497   __ sub(t0, t1, t0);
 498   __ andi(sp, t0, -16);
 499 
 500   // Restore expression stack pointer
 501   __ ld(esp, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize));
 502   // NULL last_sp until next java call
 503   __ sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize));
 504 
 505   // handle exceptions
 506   {
 507     Label L;
 508     __ ld(t0, Address(xthread, Thread::pending_exception_offset()));
 509     __ beqz(t0, L);
 510     __ call_VM(noreg,
 511                CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));
 512     __ should_not_reach_here();
 513     __ bind(L);
 514   }
 515 
 516   if (continuation == NULL) {
 517     __ dispatch_next(state, step);
 518   } else {
 519     __ jump_to_entry(continuation);
 520   }
 521   return entry;
 522 }
 523 
 524 address TemplateInterpreterGenerator::generate_result_handler_for(BasicType type) {
 525   address entry = __ pc();
 526   if (type == T_OBJECT) {
 527     // retrieve result from frame
 528     __ ld(x10, Address(fp, frame::interpreter_frame_oop_temp_offset * wordSize));
 529     // and verify it
 530     __ verify_oop(x10);
 531   } else {
 532    __ cast_primitive_type(type, x10);
 533   }
 534 
 535   __ ret();                                  // return from result handler
 536   return entry;
 537 }
 538 
 539 address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state,
 540                                                                 address runtime_entry) {
 541   assert_cond(runtime_entry != NULL);
 542   address entry = __ pc();
 543   __ push(state);
 544   __ call_VM(noreg, runtime_entry);
 545   __ fence(0xf, 0xf);
 546   __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));
 547   return entry;
 548 }
 549 
 550 // Helpers for commoning out cases in the various type of method entries.
 551 //
 552 
 553 
 554 // increment invocation count & check for overflow
 555 //
 556 // Note: checking for negative value instead of overflow
 557 //       so we have a 'sticky' overflow test
 558 //
 559 // xmethod: method
 560 //
 561 void TemplateInterpreterGenerator::generate_counter_incr(Label* overflow) {
 562   Label done;
 563   // Note: In tiered we increment either counters in Method* or in MDO depending if we're profiling or not.
 564   int increment = InvocationCounter::count_increment;
 565   Label no_mdo;
 566   if (ProfileInterpreter) {
 567     // Are we profiling?
 568     __ ld(x10, Address(xmethod, Method::method_data_offset()));
 569     __ beqz(x10, no_mdo);
 570     // Increment counter in the MDO
 571     const Address mdo_invocation_counter(x10, in_bytes(MethodData::invocation_counter_offset()) +
 572                                          in_bytes(InvocationCounter::counter_offset()));
 573     const Address mask(x10, in_bytes(MethodData::invoke_mask_offset()));
 574     __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, t0, t1, false, overflow);
 575     __ j(done);
 576   }
 577   __ bind(no_mdo);
 578   // Increment counter in MethodCounters
 579   const Address invocation_counter(t1,
 580                                    MethodCounters::invocation_counter_offset() +
 581                                    InvocationCounter::counter_offset());
 582   __ get_method_counters(xmethod, t1, done);
 583   const Address mask(t1, in_bytes(MethodCounters::invoke_mask_offset()));
 584   __ increment_mask_and_jump(invocation_counter, increment, mask, t0, x11, false, overflow);
 585   __ bind(done);
 586 }
 587 
 588 void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) {
 589   __ mv(c_rarg1, zr);
 590   __ call_VM(noreg,
 591              CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), c_rarg1);
 592   __ j(do_continue);
 593 }
 594 
 595 // See if we've got enough room on the stack for locals plus overhead
 596 // below JavaThread::stack_overflow_limit(). If not, throw a StackOverflowError
 597 // without going through the signal handler, i.e., reserved and yellow zones
 598 // will not be made usable. The shadow zone must suffice to handle the
 599 // overflow.
 600 // The expression stack grows down incrementally, so the normal guard
 601 // page mechanism will work for that.
 602 //
 603 // NOTE: Since the additional locals are also always pushed (wasn't
 604 // obvious in generate_method_entry) so the guard should work for them
 605 // too.
 606 //
 607 // Args:
 608 //      x13: number of additional locals this frame needs (what we must check)
 609 //      xmethod: Method*
 610 //
 611 // Kills:
 612 //      x10
 613 void TemplateInterpreterGenerator::generate_stack_overflow_check(void) {
 614 
 615   // monitor entry size: see picture of stack set
 616   // (generate_method_entry) and frame_amd64.hpp
 617   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 618 
 619   // total overhead size: entry_size + (saved fp through expr stack
 620   // bottom).  be sure to change this if you add/subtract anything
 621   // to/from the overhead area
 622   const int overhead_size =
 623     -(frame::interpreter_frame_initial_sp_offset * wordSize) + entry_size;
 624 
 625   const int page_size = os::vm_page_size();
 626 
 627   Label after_frame_check;
 628 
 629   // see if the frame is greater than one page in size. If so,
 630   // then we need to verify there is enough stack space remaining
 631   // for the additional locals.
 632   __ mv(t0, (page_size - overhead_size) / Interpreter::stackElementSize);
 633   __ bleu(x13, t0, after_frame_check);
 634 
 635   // compute sp as if this were going to be the last frame on
 636   // the stack before the red zone
 637 
 638   // locals + overhead, in bytes
 639   __ mv(x10, overhead_size);
 640   __ slli(t0, x13, Interpreter::logStackElementSize);
 641   __ add(x10, x10, t0);  // 2 slots per parameter.
 642 
 643   const Address stack_limit(xthread, JavaThread::stack_overflow_limit_offset());
 644   __ ld(t0, stack_limit);
 645 
 646 #ifdef ASSERT
 647   Label limit_okay;
 648   // Verify that thread stack limit is non-zero.
 649   __ bnez(t0, limit_okay);
 650   __ stop("stack overflow limit is zero");
 651   __ bind(limit_okay);
 652 #endif
 653 
 654   // Add stack limit to locals.
 655   __ add(x10, x10, t0);
 656 
 657   // Check against the current stack bottom.
 658   __ bgtu(sp, x10, after_frame_check);
 659 
 660   // Remove the incoming args, peeling the machine SP back to where it
 661   // was in the caller.  This is not strictly necessary, but unless we
 662   // do so the stack frame may have a garbage FP; this ensures a
 663   // correct call stack that we can always unwind.  The ANDI should be
 664   // unnecessary because the sender SP in x30 is always aligned, but
 665   // it doesn't hurt.
 666   __ andi(sp, x30, -16);
 667 
 668   // Note: the restored frame is not necessarily interpreted.
 669   // Use the shared runtime version of the StackOverflowError.
 670   assert(StubRoutines::throw_StackOverflowError_entry() != NULL, "stub not yet generated");
 671   __ far_jump(RuntimeAddress(StubRoutines::throw_StackOverflowError_entry()));
 672 
 673   // all done with frame size check
 674   __ bind(after_frame_check);
 675 }
 676 
 677 // Allocate monitor and lock method (asm interpreter)
 678 //
 679 // Args:
 680 //      xmethod: Method*
 681 //      xlocals: locals
 682 //
 683 // Kills:
 684 //      x10
 685 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, ...(param regs)
 686 //      t0, t1 (temporary regs)
 687 void TemplateInterpreterGenerator::lock_method() {
 688   // synchronize method
 689   const Address access_flags(xmethod, Method::access_flags_offset());
 690   const Address monitor_block_top(fp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
 691   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 692 
 693 #ifdef ASSERT
 694   __ lwu(x10, access_flags);
 695   __ verify_access_flags(x10, JVM_ACC_SYNCHRONIZED, "method doesn't need synchronization", false);
 696 #endif // ASSERT
 697 
 698   // get synchronization object
 699   {
 700     Label done;
 701     __ lwu(x10, access_flags);
 702     __ andi(t0, x10, JVM_ACC_STATIC);
 703     // get receiver (assume this is frequent case)
 704     __ ld(x10, Address(xlocals, Interpreter::local_offset_in_bytes(0)));
 705     __ beqz(t0, done);
 706     __ load_mirror(x10, xmethod);
 707 
 708 #ifdef ASSERT
 709     {
 710       Label L;
 711       __ bnez(x10, L);
 712       __ stop("synchronization object is NULL");
 713       __ bind(L);
 714     }
 715 #endif // ASSERT
 716 
 717     __ bind(done);
 718   }
 719 
 720   // add space for monitor & lock
 721   __ add(sp, sp, - entry_size); // add space for a monitor entry
 722   __ add(esp, esp, - entry_size);
 723   __ mv(t0, esp);
 724   __ sd(t0, monitor_block_top);  // set new monitor block top
 725   // store object
 726   __ sd(x10, Address(esp, BasicObjectLock::obj_offset_in_bytes()));
 727   __ mv(c_rarg1, esp); // object address
 728   __ lock_object(c_rarg1);
 729 }
 730 
 731 // Generate a fixed interpreter frame. This is identical setup for
 732 // interpreted methods and for native methods hence the shared code.
 733 //
 734 // Args:
 735 //      lr: return address
 736 //      xmethod: Method*
 737 //      xlocals: pointer to locals
 738 //      xcpool: cp cache
 739 //      stack_pointer: previous sp
 740 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
 741   // initialize fixed part of activation frame
 742   if (native_call) {
 743     __ add(esp, sp, - 14 * wordSize);
 744     __ mv(xbcp, zr);
 745     __ add(sp, sp, - 14 * wordSize);
 746     // add 2 zero-initialized slots for native calls
 747     __ sd(zr, Address(sp, 13 * wordSize));
 748     __ sd(zr, Address(sp, 12 * wordSize));
 749   } else {
 750     __ add(esp, sp, - 12 * wordSize);
 751     __ ld(t0, Address(xmethod, Method::const_offset()));     // get ConstMethod
 752     __ add(xbcp, t0, in_bytes(ConstMethod::codes_offset())); // get codebase
 753     __ add(sp, sp, - 12 * wordSize);
 754   }
 755   __ sd(xbcp, Address(sp, wordSize));
 756   __ sd(esp, Address(sp, 0));
 757 
 758   if (ProfileInterpreter) {
 759     Label method_data_continue;
 760     __ ld(t0, Address(xmethod, Method::method_data_offset()));
 761     __ beqz(t0, method_data_continue);
 762     __ la(t0, Address(t0, in_bytes(MethodData::data_offset())));
 763     __ bind(method_data_continue);
 764   }
 765 
 766   __ sd(xmethod, Address(sp, 7 * wordSize));
 767   __ sd(ProfileInterpreter ? t0 : zr, Address(sp, 6 * wordSize));
 768 
 769   // Get mirror and store it in the frame as GC root for this Method*
 770   __ load_mirror(t2, xmethod);
 771   __ sd(zr, Address(sp, 5 * wordSize));
 772   __ sd(t2, Address(sp, 4 * wordSize));
 773 
 774   __ ld(xcpool, Address(xmethod, Method::const_offset()));
 775   __ ld(xcpool, Address(xcpool, ConstMethod::constants_offset()));
 776   __ ld(xcpool, Address(xcpool, ConstantPool::cache_offset_in_bytes()));
 777   __ sd(xcpool, Address(sp, 3 * wordSize));
 778   __ sd(xlocals, Address(sp, 2 * wordSize));
 779 
 780   __ sd(lr, Address(sp, 11 * wordSize));
 781   __ sd(fp, Address(sp, 10 * wordSize));
 782   __ la(fp, Address(sp, 10 * wordSize));
 783 
 784   // set sender sp
 785   // leave last_sp as null
 786   __ sd(x30, Address(sp, 9 * wordSize));
 787   __ sd(zr, Address(sp, 8 * wordSize));
 788 
 789   // Move SP out of the way
 790   if (!native_call) {
 791     __ ld(t0, Address(xmethod, Method::const_offset()));
 792     __ lhu(t0, Address(t0, ConstMethod::max_stack_offset()));
 793     __ add(t0, t0, frame::interpreter_frame_monitor_size() + 2);
 794     __ slli(t0, t0, 3);
 795     __ sub(t0, sp, t0);
 796     __ andi(sp, t0, -16);
 797   }
 798 }
 799 
 800 // End of helpers
 801 
 802 // Various method entries
 803 //------------------------------------------------------------------------------------------------------------------------
 804 //
 805 //
 806 
 807 // Method entry for java.lang.ref.Reference.get.
 808 address TemplateInterpreterGenerator::generate_Reference_get_entry(void) {
 809   // Code: _aload_0, _getfield, _areturn
 810   // parameter size = 1
 811   //
 812   // The code that gets generated by this routine is split into 2 parts:
 813   //    1. The "intrinsified" code for G1 (or any SATB based GC),
 814   //    2. The slow path - which is an expansion of the regular method entry.
 815   //
 816   // Notes:-
 817   // * In the G1 code we do not check whether we need to block for
 818   //   a safepoint. If G1 is enabled then we must execute the specialized
 819   //   code for Reference.get (except when the Reference object is null)
 820   //   so that we can log the value in the referent field with an SATB
 821   //   update buffer.
 822   //   If the code for the getfield template is modified so that the
 823   //   G1 pre-barrier code is executed when the current method is
 824   //   Reference.get() then going through the normal method entry
 825   //   will be fine.
 826   // * The G1 code can, however, check the receiver object (the instance
 827   //   of java.lang.Reference) and jump to the slow path if null. If the
 828   //   Reference object is null then we obviously cannot fetch the referent
 829   //   and so we don't need to call the G1 pre-barrier. Thus we can use the
 830   //   regular method entry code to generate the NPE.
 831   //
 832   // This code is based on generate_accessor_entry.
 833   //
 834   // xmethod: Method*
 835   // x30: senderSP must preserve for slow path, set SP to it on fast path
 836 
 837   // LR is live.  It must be saved around calls.
 838 
 839   address entry = __ pc();
 840 
 841   const int referent_offset = java_lang_ref_Reference::referent_offset();
 842   guarantee(referent_offset > 0, "referent offset not initialized");
 843 
 844   Label slow_path;
 845   const Register local_0 = c_rarg0;
 846   // Check if local 0 != NULL
 847   // If the receiver is null then it is OK to jump to the slow path.
 848   __ ld(local_0, Address(esp, 0));
 849   __ beqz(local_0, slow_path);
 850 
 851   __ mv(x9, x30);   // Move senderSP to a callee-saved register
 852 
 853   // Load the value of the referent field.
 854   const Address field_address(local_0, referent_offset);
 855   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 856   bs->load_at(_masm, IN_HEAP | ON_WEAK_OOP_REF, T_OBJECT, local_0, field_address, /*tmp1*/ t1, /*tmp2*/ t0);
 857 
 858   // areturn
 859   __ andi(sp, x9, -16);  // done with stack
 860   __ ret();
 861 
 862   // generate a vanilla interpreter entry as the slow path
 863   __ bind(slow_path);
 864   __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals));
 865   return entry;
 866 }
 867 
 868 /**
 869  * Method entry for static native methods:
 870  *   int java.util.zip.CRC32.update(int crc, int b)
 871  */
 872 address TemplateInterpreterGenerator::generate_CRC32_update_entry() {
 873   // TODO: Unimplemented generate_CRC32_update_entry
 874   return 0;
 875 }
 876 
 877 /**
 878  * Method entry for static native methods:
 879  *   int java.util.zip.CRC32.updateBytes(int crc, byte[] b, int off, int len)
 880  *   int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
 881  */
 882 address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) {
 883   // TODO: Unimplemented generate_CRC32_updateBytes_entry
 884   return 0;
 885 }
 886 
 887 /**
 888  * Method entry for intrinsic-candidate (non-native) methods:
 889  *   int java.util.zip.CRC32C.updateBytes(int crc, byte[] b, int off, int end)
 890  *   int java.util.zip.CRC32C.updateDirectByteBuffer(int crc, long buf, int off, int end)
 891  * Unlike CRC32, CRC32C does not have any methods marked as native
 892  * CRC32C also uses an "end" variable instead of the length variable CRC32 uses
 893  */
 894 address TemplateInterpreterGenerator::generate_CRC32C_updateBytes_entry(AbstractInterpreter::MethodKind kind) {
 895   // TODO: Unimplemented generate_CRC32C_updateBytes_entry
 896   return 0;
 897 }
 898 
 899 void TemplateInterpreterGenerator::bang_stack_shadow_pages(bool native_call) {
 900   // Bang each page in the shadow zone. We can't assume it's been done for
 901   // an interpreter frame with greater than a page of locals, so each page
 902   // needs to be checked.  Only true for non-native.
 903   const int n_shadow_pages = StackOverflow::stack_shadow_zone_size() / os::vm_page_size();
 904   const int start_page = native_call ? n_shadow_pages : 1;
 905   const int page_size = os::vm_page_size();
 906   for (int pages = start_page; pages <= n_shadow_pages ; pages++) {
 907     __ sub(t1, sp, pages * page_size);
 908     __ sd(zr, Address(t1));
 909   }
 910 }
 911 
 912 // Interpreter stub for calling a native method. (asm interpreter)
 913 // This sets up a somewhat different looking stack for calling the
 914 // native method than the typical interpreter frame setup.
 915 address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
 916   // determine code generation flags
 917   bool inc_counter  = UseCompiler || CountCompiledCalls || LogTouchedMethods;
 918 
 919   // x11: Method*
 920   // x30: sender sp
 921 
 922   address entry_point = __ pc();
 923 
 924   const Address constMethod       (xmethod, Method::const_offset());
 925   const Address access_flags      (xmethod, Method::access_flags_offset());
 926   const Address size_of_parameters(x12, ConstMethod::
 927                                    size_of_parameters_offset());
 928 
 929   // get parameter size (always needed)
 930   __ ld(x12, constMethod);
 931   __ load_unsigned_short(x12, size_of_parameters);
 932 
 933   // Native calls don't need the stack size check since they have no
 934   // expression stack and the arguments are already on the stack and
 935   // we only add a handful of words to the stack.
 936 
 937   // xmethod: Method*
 938   // x12: size of parameters
 939   // x30: sender sp
 940 
 941   // for natives the size of locals is zero
 942 
 943   // compute beginning of parameters (xlocals)
 944   __ slli(xlocals, x12, 3);
 945   __ add(xlocals, esp, xlocals);
 946   __ addi(xlocals, xlocals, -wordSize);
 947 
 948   // Pull SP back to minimum size: this avoids holes in the stack
 949   __ andi(sp, esp, -16);
 950 
 951   // initialize fixed part of activation frame
 952   generate_fixed_frame(true);
 953 
 954   // make sure method is native & not abstract
 955 #ifdef ASSERT
 956   __ lwu(x10, access_flags);
 957   __ verify_access_flags(x10, JVM_ACC_NATIVE, "tried to execute non-native method as native", false);
 958   __ verify_access_flags(x10, JVM_ACC_ABSTRACT, "tried to execute abstract method in interpreter");
 959 #endif
 960 
 961   // Since at this point in the method invocation the exception
 962   // handler would try to exit the monitor of synchronized methods
 963   // which hasn't been entered yet, we set the thread local variable
 964   // _do_not_unlock_if_synchronized to true. The remove_activation
 965   // will check this flag.
 966 
 967   const Address do_not_unlock_if_synchronized(xthread,
 968                                               in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
 969   __ mv(t1, true);
 970   __ sb(t1, do_not_unlock_if_synchronized);
 971 
 972   // increment invocation count & check for overflow
 973   Label invocation_counter_overflow;
 974   if (inc_counter) {
 975     generate_counter_incr(&invocation_counter_overflow);
 976   }
 977 
 978   Label continue_after_compile;
 979   __ bind(continue_after_compile);
 980 
 981   bang_stack_shadow_pages(true);
 982 
 983   // reset the _do_not_unlock_if_synchronized flag
 984   __ sb(zr, do_not_unlock_if_synchronized);
 985 
 986   // check for synchronized methods
 987   // Must happen AFTER invocation_counter check and stack overflow check,
 988   // so method is not locked if overflows.
 989   if (synchronized) {
 990     lock_method();
 991   } else {
 992     // no synchronization necessary
 993 #ifdef ASSERT
 994     __ lwu(x10, access_flags);
 995     __ verify_access_flags(x10, JVM_ACC_SYNCHRONIZED, "method needs synchronization");
 996 #endif
 997   }
 998 
 999   // start execution
1000 #ifdef ASSERT
1001   __ verify_frame_setup();
1002 #endif
1003 
1004   // jvmti support
1005   __ notify_method_entry();
1006 
1007   // work registers
1008   const Register t = x18;
1009   const Register result_handler = x19;
1010 
1011   // allocate space for parameters
1012   __ ld(t, Address(xmethod, Method::const_offset()));
1013   __ load_unsigned_short(t, Address(t, ConstMethod::size_of_parameters_offset()));
1014 
1015   __ slli(t, t, Interpreter::logStackElementSize);
1016   __ sub(x30, esp, t);
1017   __ andi(sp, x30, -16);
1018   __ mv(esp, x30);
1019 
1020   // get signature handler
1021   {
1022     Label L;
1023     __ ld(t, Address(xmethod, Method::signature_handler_offset()));
1024     __ bnez(t, L);
1025     __ call_VM(noreg,
1026                CAST_FROM_FN_PTR(address,
1027                                 InterpreterRuntime::prepare_native_call),
1028                xmethod);
1029     __ ld(t, Address(xmethod, Method::signature_handler_offset()));
1030     __ bind(L);
1031   }
1032 
1033   // call signature handler
1034   assert(InterpreterRuntime::SignatureHandlerGenerator::from() == xlocals,
1035          "adjust this code");
1036   assert(InterpreterRuntime::SignatureHandlerGenerator::to() == sp,
1037          "adjust this code");
1038   assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == t0,
1039           "adjust this code");
1040 
1041   // The generated handlers do not touch xmethod (the method).
1042   // However, large signatures cannot be cached and are generated
1043   // each time here.  The slow-path generator can do a GC on return,
1044   // so we must reload it after the call.
1045   __ jalr(t);
1046   __ get_method(xmethod);        // slow path can do a GC, reload xmethod
1047 
1048 
1049   // result handler is in x10
1050   // set result handler
1051   __ mv(result_handler, x10);
1052   // pass mirror handle if static call
1053   {
1054     Label L;
1055     __ lwu(t, Address(xmethod, Method::access_flags_offset()));
1056     __ andi(t0, t, JVM_ACC_STATIC);
1057     __ beqz(t0, L);
1058     // get mirror
1059     __ load_mirror(t, xmethod);
1060     // copy mirror into activation frame
1061     __ sd(t, Address(fp, frame::interpreter_frame_oop_temp_offset * wordSize));
1062     // pass handle to mirror
1063     __ addi(c_rarg1, fp, frame::interpreter_frame_oop_temp_offset * wordSize);
1064     __ bind(L);
1065   }
1066 
1067   // get native function entry point in x28
1068   {
1069     Label L;
1070     __ ld(x28, Address(xmethod, Method::native_function_offset()));
1071     address unsatisfied = (SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
1072     __ mv(t1, unsatisfied);
1073     __ ld(t1, t1);
1074     __ bne(x28, t1, L);
1075     __ call_VM(noreg,
1076                CAST_FROM_FN_PTR(address,
1077                                 InterpreterRuntime::prepare_native_call),
1078                xmethod);
1079     __ get_method(xmethod);
1080     __ ld(x28, Address(xmethod, Method::native_function_offset()));
1081     __ bind(L);
1082   }
1083 
1084   // pass JNIEnv
1085   __ add(c_rarg0, xthread, in_bytes(JavaThread::jni_environment_offset()));
1086 
1087   // It is enough that the pc() points into the right code
1088   // segment. It does not have to be the correct return pc.
1089   Label native_return;
1090   __ set_last_Java_frame(esp, fp, native_return, x30);
1091 
1092   // change thread state
1093 #ifdef ASSERT
1094   {
1095     Label L;
1096     __ lwu(t, Address(xthread, JavaThread::thread_state_offset()));
1097     __ addi(t0, zr, (u1)_thread_in_Java);
1098     __ beq(t, t0, L);
1099     __ stop("Wrong thread state in native stub");
1100     __ bind(L);
1101   }
1102 #endif
1103 
1104   // Change state to native
1105   __ la(t1, Address(xthread, JavaThread::thread_state_offset()));
1106   __ mv(t0, _thread_in_native);
1107   __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
1108   __ sw(t0, Address(t1));
1109 
1110   // Call the native method.
1111   __ jalr(x28);
1112   __ bind(native_return);
1113   __ get_method(xmethod);
1114   // result potentially in x10 or f10
1115 
1116   // make room for the pushes we're about to do
1117   __ sub(t0, esp, 4 * wordSize);
1118   __ andi(sp, t0, -16);
1119 
1120   // NOTE: The order of these pushes is known to frame::interpreter_frame_result
1121   // in order to extract the result of a method call. If the order of these
1122   // pushes change or anything else is added to the stack then the code in
1123   // interpreter_frame_result must also change.
1124   __ push(dtos);
1125   __ push(ltos);
1126 
1127   // change thread state
1128   // Force all preceding writes to be observed prior to thread state change
1129   __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
1130 
1131   __ mv(t0, _thread_in_native_trans);
1132   __ sw(t0, Address(xthread, JavaThread::thread_state_offset()));
1133 
1134   // Force this write out before the read below
1135   __ membar(MacroAssembler::AnyAny);
1136 
1137   // check for safepoint operation in progress and/or pending suspend requests
1138   {
1139     Label L, Continue;
1140 
1141     // We need an acquire here to ensure that any subsequent load of the
1142     // global SafepointSynchronize::_state flag is ordered after this load
1143     // of the thread-local polling word. We don't want this poll to
1144     // return false (i.e. not safepointing) and a later poll of the global
1145     // SafepointSynchronize::_state spuriously to return true.
1146     //
1147     // This is to avoid a race when we're in a native->Java transition
1148     // racing the code which wakes up from a safepoint.
1149     __ safepoint_poll(L, true /* at_return */, true /* acquire */, false /* in_nmethod */);
1150     __ lwu(t1, Address(xthread, JavaThread::suspend_flags_offset()));
1151     __ beqz(t1, Continue);
1152     __ bind(L);
1153 
1154     // Don't use call_VM as it will see a possible pending exception
1155     // and forward it and never return here preventing us from
1156     // clearing _last_native_pc down below. So we do a runtime call by
1157     // hand.
1158     //
1159     __ mv(c_rarg0, xthread);
1160     __ mv(t1, CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
1161     __ jalr(t1);
1162     __ get_method(xmethod);
1163     __ reinit_heapbase();
1164     __ bind(Continue);
1165   }
1166 
1167   // change thread state
1168   // Force all preceding writes to be observed prior to thread state change
1169   __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
1170 
1171   __ mv(t0, _thread_in_Java);
1172   __ sw(t0, Address(xthread, JavaThread::thread_state_offset()));
1173 
1174   // reset_last_Java_frame
1175   __ reset_last_Java_frame(true);
1176 
1177   if (CheckJNICalls) {
1178     // clear_pending_jni_exception_check
1179     __ sd(zr, Address(xthread, JavaThread::pending_jni_exception_check_fn_offset()));
1180   }
1181 
1182   // reset handle block
1183   __ ld(t, Address(xthread, JavaThread::active_handles_offset()));
1184   __ sd(zr, Address(t, JNIHandleBlock::top_offset_in_bytes()));
1185 
1186   // If result is an oop unbox and store it in frame where gc will see it
1187   // and result handler will pick it up
1188 
1189   {
1190     Label no_oop;
1191     __ la(t, ExternalAddress(AbstractInterpreter::result_handler(T_OBJECT)));
1192     __ bne(t, result_handler, no_oop);
1193     // Unbox oop result, e.g. JNIHandles::resolve result.
1194     __ pop(ltos);
1195     __ resolve_jobject(x10, xthread, t);
1196     __ sd(x10, Address(fp, frame::interpreter_frame_oop_temp_offset * wordSize));
1197     // keep stack depth as expected by pushing oop which will eventually be discarded
1198     __ push(ltos);
1199     __ bind(no_oop);
1200   }
1201 
1202   {
1203     Label no_reguard;
1204     __ lwu(t0, Address(xthread, in_bytes(JavaThread::stack_guard_state_offset())));
1205     __ addi(t1, zr, (u1)StackOverflow::stack_guard_yellow_reserved_disabled);
1206     __ bne(t0, t1, no_reguard);
1207 
1208     __ pusha(); // only save smashed registers
1209     __ mv(c_rarg0, xthread);
1210     __ mv(t1, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
1211     __ jalr(t1);
1212     __ popa(); // only restore smashed registers
1213     __ bind(no_reguard);
1214   }
1215 
1216   // The method register is junk from after the thread_in_native transition
1217   // until here.  Also can't call_VM until the bcp has been
1218   // restored.  Need bcp for throwing exception below so get it now.
1219   __ get_method(xmethod);
1220 
1221   // restore bcp to have legal interpreter frame, i.e., bci == 0 <=>
1222   // xbcp == code_base()
1223   __ ld(xbcp, Address(xmethod, Method::const_offset()));   // get ConstMethod*
1224   __ add(xbcp, xbcp, in_bytes(ConstMethod::codes_offset()));          // get codebase
1225   // handle exceptions (exception handling will handle unlocking!)
1226   {
1227     Label L;
1228     __ ld(t0, Address(xthread, Thread::pending_exception_offset()));
1229     __ beqz(t0, L);
1230     // Note: At some point we may want to unify this with the code
1231     // used in call_VM_base(); i.e., we should use the
1232     // StubRoutines::forward_exception code. For now this doesn't work
1233     // here because the sp is not correctly set at this point.
1234     __ MacroAssembler::call_VM(noreg,
1235                                CAST_FROM_FN_PTR(address,
1236                                InterpreterRuntime::throw_pending_exception));
1237     __ should_not_reach_here();
1238     __ bind(L);
1239   }
1240 
1241   // do unlocking if necessary
1242   {
1243     Label L;
1244     __ lwu(t, Address(xmethod, Method::access_flags_offset()));
1245     __ andi(t0, t, JVM_ACC_SYNCHRONIZED);
1246     __ beqz(t0, L);
1247     // the code below should be shared with interpreter macro
1248     // assembler implementation
1249     {
1250       Label unlock;
1251       // BasicObjectLock will be first in list, since this is a
1252       // synchronized method. However, need to check that the object
1253       // has not been unlocked by an explicit monitorexit bytecode.
1254 
1255       // monitor expect in c_rarg1 for slow unlock path
1256       __ la(c_rarg1, Address(fp,   // address of first monitor
1257                              (intptr_t)(frame::interpreter_frame_initial_sp_offset *
1258                                         wordSize - sizeof(BasicObjectLock))));
1259 
1260       __ ld(t, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
1261       __ bnez(t, unlock);
1262 
1263       // Entry already unlocked, need to throw exception
1264       __ MacroAssembler::call_VM(noreg,
1265                                  CAST_FROM_FN_PTR(address,
1266                                                   InterpreterRuntime::throw_illegal_monitor_state_exception));
1267       __ should_not_reach_here();
1268 
1269       __ bind(unlock);
1270       __ unlock_object(c_rarg1);
1271     }
1272     __ bind(L);
1273   }
1274 
1275   // jvmti support
1276   // Note: This must happen _after_ handling/throwing any exceptions since
1277   //       the exception handler code notifies the runtime of method exits
1278   //       too. If this happens before, method entry/exit notifications are
1279   //       not properly paired (was bug - gri 11/22/99).
1280   __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI);
1281 
1282   __ pop(ltos);
1283   __ pop(dtos);
1284 
1285   __ jalr(result_handler);
1286 
1287   // remove activation
1288   __ ld(esp, Address(fp, frame::interpreter_frame_sender_sp_offset * wordSize)); // get sender sp
1289   // remove frame anchor
1290   __ leave();
1291 
1292   // restore sender sp
1293   __ mv(sp, esp);
1294 
1295   __ ret();
1296 
1297   if (inc_counter) {
1298     // Handle overflow of counter and compile method
1299     __ bind(invocation_counter_overflow);
1300     generate_counter_overflow(continue_after_compile);
1301   }
1302 
1303   return entry_point;
1304 }
1305 
1306 //
1307 // Generic interpreted method entry to (asm) interpreter
1308 //
1309 address TemplateInterpreterGenerator::generate_normal_entry(bool synchronized) {
1310 
1311   // determine code generation flags
1312   const bool inc_counter  = UseCompiler || CountCompiledCalls || LogTouchedMethods;
1313 
1314   // t0: sender sp
1315   address entry_point = __ pc();
1316 
1317   const Address constMethod(xmethod, Method::const_offset());
1318   const Address access_flags(xmethod, Method::access_flags_offset());
1319   const Address size_of_parameters(x13,
1320                                    ConstMethod::size_of_parameters_offset());
1321   const Address size_of_locals(x13, ConstMethod::size_of_locals_offset());
1322 
1323   // get parameter size (always needed)
1324   // need to load the const method first
1325   __ ld(x13, constMethod);
1326   __ load_unsigned_short(x12, size_of_parameters);
1327 
1328   // x12: size of parameters
1329 
1330   __ load_unsigned_short(x13, size_of_locals); // get size of locals in words
1331   __ sub(x13, x13, x12); // x13 = no. of additional locals
1332 
1333   // see if we've got enough room on the stack for locals plus overhead.
1334   generate_stack_overflow_check();
1335 
1336   // compute beginning of parameters (xlocals)
1337   __ slli(t1, x12, 3);
1338   __ add(xlocals, esp, t1);
1339   __ add(xlocals, xlocals, -wordSize);
1340 
1341   // Make room for additional locals
1342   __ slli(t1, x13, 3);
1343   __ sub(t0, esp, t1);
1344 
1345   // Padding between locals and fixed part of activation frame to ensure
1346   // SP is always 16-byte aligned.
1347   __ andi(sp, t0, -16);
1348 
1349   // x13 - # of additional locals
1350   // allocate space for locals
1351   // explicitly initialize locals
1352   {
1353     Label exit, loop;
1354     __ blez(x13, exit); // do nothing if x13 <= 0
1355     __ bind(loop);
1356     __ sd(zr, Address(t0));
1357     __ add(t0, t0, wordSize);
1358     __ add(x13, x13, -1); // until everything initialized
1359     __ bnez(x13, loop);
1360     __ bind(exit);
1361   }
1362 
1363   // And the base dispatch table
1364   __ get_dispatch();
1365 
1366   // initialize fixed part of activation frame
1367   generate_fixed_frame(false);
1368 
1369   // make sure method is not native & not abstract
1370 #ifdef ASSERT
1371   __ lwu(x10, access_flags);
1372   __ verify_access_flags(x10, JVM_ACC_NATIVE, "tried to execute native method as non-native");
1373   __ verify_access_flags(x10, JVM_ACC_ABSTRACT, "tried to execute abstract method in interpreter");
1374 #endif
1375 
1376   // Since at this point in the method invocation the exception
1377   // handler would try to exit the monitor of synchronized methods
1378   // which hasn't been entered yet, we set the thread local variable
1379   // _do_not_unlock_if_synchronized to true. The remove_activation
1380   // will check this flag.
1381 
1382   const Address do_not_unlock_if_synchronized(xthread,
1383                                               in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1384   __ mv(t1, true);
1385   __ sb(t1, do_not_unlock_if_synchronized);
1386 
1387   Label no_mdp;
1388   const Register mdp = x13;
1389   __ ld(mdp, Address(xmethod, Method::method_data_offset()));
1390   __ beqz(mdp, no_mdp);
1391   __ add(mdp, mdp, in_bytes(MethodData::data_offset()));
1392   __ profile_parameters_type(mdp, x11, x12, x14); // use x11, x12, x14 as tmp registers
1393   __ bind(no_mdp);
1394 
1395   // increment invocation count & check for overflow
1396   Label invocation_counter_overflow;
1397   if (inc_counter) {
1398     generate_counter_incr(&invocation_counter_overflow);
1399   }
1400 
1401   Label continue_after_compile;
1402   __ bind(continue_after_compile);
1403 
1404   bang_stack_shadow_pages(false);
1405 
1406   // reset the _do_not_unlock_if_synchronized flag
1407   __ sb(zr, do_not_unlock_if_synchronized);
1408 
1409   // check for synchronized methods
1410   // Must happen AFTER invocation_counter check and stack overflow check,
1411   // so method is not locked if overflows.
1412   if (synchronized) {
1413     // Allocate monitor and lock method
1414     lock_method();
1415   } else {
1416     // no synchronization necessary
1417 #ifdef ASSERT
1418     __ lwu(x10, access_flags);
1419     __ verify_access_flags(x10, JVM_ACC_SYNCHRONIZED, "method needs synchronization");
1420 #endif
1421   }
1422 
1423   // start execution
1424 #ifdef ASSERT
1425   __ verify_frame_setup();
1426 #endif
1427 
1428   // jvmti support
1429   __ notify_method_entry();
1430 
1431   __ dispatch_next(vtos);
1432 
1433   // invocation counter overflow
1434   if (inc_counter) {
1435     // Handle overflow of counter and compile method
1436     __ bind(invocation_counter_overflow);
1437     generate_counter_overflow(continue_after_compile);
1438   }
1439 
1440   return entry_point;
1441 }
1442 
1443 //-----------------------------------------------------------------------------
1444 // Exceptions
1445 
1446 void TemplateInterpreterGenerator::generate_throw_exception() {
1447   // Entry point in previous activation (i.e., if the caller was
1448   // interpreted)
1449   Interpreter::_rethrow_exception_entry = __ pc();
1450   // Restore sp to interpreter_frame_last_sp even though we are going
1451   // to empty the expression stack for the exception processing.
1452   __ sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize));
1453   // x10: exception
1454   // x13: return address/pc that threw exception
1455   __ restore_bcp();    // xbcp points to call/send
1456   __ restore_locals();
1457   __ restore_constant_pool_cache();
1458   __ reinit_heapbase();  // restore xheapbase as heapbase.
1459   __ get_dispatch();
1460 
1461   // Entry point for exceptions thrown within interpreter code
1462   Interpreter::_throw_exception_entry = __ pc();
1463   // If we came here via a NullPointerException on the receiver of a
1464   // method, xthread may be corrupt.
1465   __ get_method(xmethod);
1466   // expression stack is undefined here
1467   // x10: exception
1468   // xbcp: exception bcp
1469   __ verify_oop(x10);
1470   __ mv(c_rarg1, x10);
1471 
1472   // expression stack must be empty before entering the VM in case of
1473   // an exception
1474   __ empty_expression_stack();
1475   // find exception handler address and preserve exception oop
1476   __ call_VM(x13,
1477              CAST_FROM_FN_PTR(address,
1478                           InterpreterRuntime::exception_handler_for_exception),
1479              c_rarg1);
1480 
1481   // Calculate stack limit
1482   __ ld(t0, Address(xmethod, Method::const_offset()));
1483   __ lhu(t0, Address(t0, ConstMethod::max_stack_offset()));
1484   __ add(t0, t0, frame::interpreter_frame_monitor_size() + 4);
1485   __ ld(t1, Address(fp, frame::interpreter_frame_initial_sp_offset * wordSize));
1486   __ slli(t0, t0, 3);
1487   __ sub(t0, t1, t0);
1488   __ andi(sp, t0, -16);
1489 
1490   // x10: exception handler entry point
1491   // x13: preserved exception oop
1492   // xbcp: bcp for exception handler
1493   __ push_ptr(x13); // push exception which is now the only value on the stack
1494   __ jr(x10); // jump to exception handler (may be _remove_activation_entry!)
1495 
1496   // If the exception is not handled in the current frame the frame is
1497   // removed and the exception is rethrown (i.e. exception
1498   // continuation is _rethrow_exception).
1499   //
1500   // Note: At this point the bci is still the bxi for the instruction
1501   // which caused the exception and the expression stack is
1502   // empty. Thus, for any VM calls at this point, GC will find a legal
1503   // oop map (with empty expression stack).
1504 
1505   //
1506   // JVMTI PopFrame support
1507   //
1508 
1509   Interpreter::_remove_activation_preserving_args_entry = __ pc();
1510   __ empty_expression_stack();
1511   // Set the popframe_processing bit in pending_popframe_condition
1512   // indicating that we are currently handling popframe, so that
1513   // call_VMs that may happen later do not trigger new popframe
1514   // handling cycles.
1515   __ lwu(x13, Address(xthread, JavaThread::popframe_condition_offset()));
1516   __ ori(x13, x13, JavaThread::popframe_processing_bit);
1517   __ sw(x13, Address(xthread, JavaThread::popframe_condition_offset()));
1518 
1519   {
1520     // Check to see whether we are returning to a deoptimized frame.
1521     // (The PopFrame call ensures that the caller of the popped frame is
1522     // either interpreted or compiled and deoptimizes it if compiled.)
1523     // In this case, we can't call dispatch_next() after the frame is
1524     // popped, but instead must save the incoming arguments and restore
1525     // them after deoptimization has occurred.
1526     //
1527     // Note that we don't compare the return PC against the
1528     // deoptimization blob's unpack entry because of the presence of
1529     // adapter frames in C2.
1530     Label caller_not_deoptimized;
1531     __ ld(c_rarg1, Address(fp, frame::return_addr_offset * wordSize));
1532     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::interpreter_contains), c_rarg1);
1533     __ bnez(x10, caller_not_deoptimized);
1534 
1535     // Compute size of arguments for saving when returning to
1536     // deoptimized caller
1537     __ get_method(x10);
1538     __ ld(x10, Address(x10, Method::const_offset()));
1539     __ load_unsigned_short(x10, Address(x10, in_bytes(ConstMethod::
1540                                                       size_of_parameters_offset())));
1541     __ slli(x10, x10, Interpreter::logStackElementSize);
1542     __ restore_locals();
1543     __ sub(xlocals, xlocals, x10);
1544     __ add(xlocals, xlocals, wordSize);
1545     // Save these arguments
1546     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1547                                            Deoptimization::
1548                                            popframe_preserve_args),
1549                           xthread, x10, xlocals);
1550 
1551     __ remove_activation(vtos,
1552                          /* throw_monitor_exception */ false,
1553                          /* install_monitor_exception */ false,
1554                          /* notify_jvmdi */ false);
1555 
1556     // Inform deoptimization that it is responsible for restoring
1557     // these arguments
1558     __ mv(t0, JavaThread::popframe_force_deopt_reexecution_bit);
1559     __ sw(t0, Address(xthread, JavaThread::popframe_condition_offset()));
1560 
1561     // Continue in deoptimization handler
1562     __ ret();
1563 
1564     __ bind(caller_not_deoptimized);
1565   }
1566 
1567   __ remove_activation(vtos,
1568                        /* throw_monitor_exception */ false,
1569                        /* install_monitor_exception */ false,
1570                        /* notify_jvmdi */ false);
1571 
1572   // Restore the last_sp and null it out
1573   __ ld(esp, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize));
1574   __ sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize));
1575 
1576   __ restore_bcp();
1577   __ restore_locals();
1578   __ restore_constant_pool_cache();
1579   __ get_method(xmethod);
1580   __ get_dispatch();
1581 
1582   // The method data pointer was incremented already during
1583   // call profiling. We have to restore the mdp for the current bcp.
1584   if (ProfileInterpreter) {
1585     __ set_method_data_pointer_for_bcp();
1586   }
1587 
1588   // Clear the popframe condition flag
1589   __ sw(zr, Address(xthread, JavaThread::popframe_condition_offset()));
1590   assert(JavaThread::popframe_inactive == 0, "fix popframe_inactive");
1591 
1592 #if INCLUDE_JVMTI
1593   {
1594     Label L_done;
1595 
1596     __ lbu(t0, Address(xbcp, 0));
1597     __ li(t1, Bytecodes::_invokestatic);
1598     __ bne(t1, t0, L_done);
1599 
1600     // The member name argument must be restored if _invokestatic is re-executed after a PopFrame call.
1601     // Detect such a case in the InterpreterRuntime function and return the member name argument,or NULL.
1602 
1603     __ ld(c_rarg0, Address(xlocals, 0));
1604     __ call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null),c_rarg0, xmethod, xbcp);
1605 
1606     __ beqz(x10, L_done);
1607 
1608     __ sd(x10, Address(esp, 0));
1609     __ bind(L_done);
1610   }
1611 #endif // INCLUDE_JVMTI
1612 
1613   // Restore machine SP
1614   __ ld(t0, Address(xmethod, Method::const_offset()));
1615   __ lhu(t0, Address(t0, ConstMethod::max_stack_offset()));
1616   __ add(t0, t0, frame::interpreter_frame_monitor_size() + 4);
1617   __ ld(t1, Address(fp, frame::interpreter_frame_initial_sp_offset * wordSize));
1618   __ slliw(t0, t0, 3);
1619   __ sub(t0, t1, t0);
1620   __ andi(sp, t0, -16);
1621 
1622   __ dispatch_next(vtos);
1623   // end of PopFrame support
1624 
1625   Interpreter::_remove_activation_entry = __ pc();
1626 
1627   // preserve exception over this code sequence
1628   __ pop_ptr(x10);
1629   __ sd(x10, Address(xthread, JavaThread::vm_result_offset()));
1630   // remove the activation (without doing throws on illegalMonitorExceptions)
1631   __ remove_activation(vtos, false, true, false);
1632   // restore exception
1633   __ get_vm_result(x10, xthread);
1634 
1635   // In between activations - previous activation type unknown yet
1636   // compute continuation point - the continuation point expects the
1637   // following registers set up:
1638   //
1639   // x10: exception
1640   // lr: return address/pc that threw exception
1641   // sp: expression stack of caller
1642   // fp: fp of caller
1643   // FIXME: There's no point saving LR here because VM calls don't trash it
1644   __ sub(sp, sp, 2 * wordSize);
1645   __ sd(x10, Address(sp, 0));                   // save exception
1646   __ sd(lr, Address(sp, wordSize));             // save return address
1647   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1648                                          SharedRuntime::exception_handler_for_return_address),
1649                         xthread, lr);
1650   __ mv(x11, x10);                              // save exception handler
1651   __ ld(x10, Address(sp, 0));                   // restore exception
1652   __ ld(lr, Address(sp, wordSize));             // restore return address
1653   __ add(sp, sp, 2 * wordSize);
1654   // We might be returning to a deopt handler that expects x13 to
1655   // contain the exception pc
1656   __ mv(x13, lr);
1657   // Note that an "issuing PC" is actually the next PC after the call
1658   __ jr(x11);                                   // jump to exception
1659                                                 // handler of caller
1660 }
1661 
1662 //
1663 // JVMTI ForceEarlyReturn support
1664 //
1665 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state)  {
1666   address entry = __ pc();
1667 
1668   __ restore_bcp();
1669   __ restore_locals();
1670   __ empty_expression_stack();
1671   __ load_earlyret_value(state);
1672 
1673   __ ld(t0, Address(xthread, JavaThread::jvmti_thread_state_offset()));
1674   Address cond_addr(t0, JvmtiThreadState::earlyret_state_offset());
1675 
1676   // Clear the earlyret state
1677   assert(JvmtiThreadState::earlyret_inactive == 0, "should be");
1678   __ sd(zr, cond_addr);
1679 
1680   __ remove_activation(state,
1681                        false, /* throw_monitor_exception */
1682                        false, /* install_monitor_exception */
1683                        true); /* notify_jvmdi */
1684   __ ret();
1685 
1686   return entry;
1687 }
1688 // end of ForceEarlyReturn support
1689 
1690 //-----------------------------------------------------------------------------
1691 // Helper for vtos entry point generation
1692 
1693 void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t,
1694                                                          address& bep,
1695                                                          address& cep,
1696                                                          address& sep,
1697                                                          address& aep,
1698                                                          address& iep,
1699                                                          address& lep,
1700                                                          address& fep,
1701                                                          address& dep,
1702                                                          address& vep) {
1703   assert(t != NULL && t->is_valid() && t->tos_in() == vtos, "illegal template");
1704   Label L;
1705   aep = __ pc();  __ push_ptr();  __ j(L);
1706   fep = __ pc();  __ push_f();    __ j(L);
1707   dep = __ pc();  __ push_d();    __ j(L);
1708   lep = __ pc();  __ push_l();    __ j(L);
1709   bep = cep = sep =
1710   iep = __ pc();  __ push_i();
1711   vep = __ pc();
1712   __ bind(L);
1713   generate_and_dispatch(t);
1714 }
1715 
1716 //-----------------------------------------------------------------------------
1717 
1718 // Non-product code
1719 #ifndef PRODUCT
1720 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1721   address entry = __ pc();
1722 
1723   __ push_reg(lr);
1724   __ push(state);
1725   __ push_reg(RegSet::range(x10, x17) + RegSet::range(x5, x7) + RegSet::range(x28, x31), sp);
1726   __ mv(c_rarg2, x10);  // Pass itos
1727   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::trace_bytecode), c_rarg1, c_rarg2, c_rarg3);
1728   __ pop_reg(RegSet::range(x10, x17) + RegSet::range(x5, x7) + RegSet::range(x28, x31), sp);
1729   __ pop(state);
1730   __ pop_reg(lr);
1731   __ ret();                                   // return from result handler
1732 
1733   return entry;
1734 }
1735 
1736 void TemplateInterpreterGenerator::count_bytecode() {
1737   __ push_reg(t0);
1738   __ push_reg(x10);
1739   __ mv(x10, (address) &BytecodeCounter::_counter_value);
1740   __ li(t0, 1);
1741   __ amoadd_d(zr, x10, t0, Assembler::aqrl);
1742   __ pop_reg(x10);
1743   __ pop_reg(t0);
1744 }
1745 
1746 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) { ; }
1747 
1748 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) { ; }
1749 
1750 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
1751   // Call a little run-time stub to avoid blow-up for each bytecode.
1752   // The run-time runtime saves the right registers, depending on
1753   // the tosca in-state for the given template.
1754 
1755   assert(Interpreter::trace_code(t->tos_in()) != NULL, "entry must have been generated");
1756   __ jal(Interpreter::trace_code(t->tos_in()));
1757   __ reinit_heapbase();
1758 }
1759 
1760 void TemplateInterpreterGenerator::stop_interpreter_at() {
1761   Label L;
1762   __ push_reg(t0);
1763   __ mv(t0, (address) &BytecodeCounter::_counter_value);
1764   __ ld(t0, Address(t0));
1765   __ mv(t1, StopInterpreterAt);
1766   __ bne(t0, t1, L);
1767   __ ebreak();
1768   __ bind(L);
1769   __ pop_reg(t0);
1770 }
1771 
1772 #endif // !PRODUCT