1 /*
   2  * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "compiler/disassembler.hpp"
  29 #include "gc/shared/barrierSetAssembler.hpp"
  30 #include "gc/shared/collectedHeap.hpp"
  31 #include "gc/shared/tlab_globals.hpp"
  32 #include "interpreter/interp_masm.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "interpreter/interpreterRuntime.hpp"
  35 #include "interpreter/templateTable.hpp"
  36 #include "memory/universe.hpp"
  37 #include "oops/method.inline.hpp"
  38 #include "oops/methodData.hpp"
  39 #include "oops/objArrayKlass.hpp"
  40 #include "oops/oop.inline.hpp"
  41 #include "oops/resolvedFieldEntry.hpp"
  42 #include "oops/resolvedIndyEntry.hpp"
  43 #include "oops/resolvedMethodEntry.hpp"
  44 #include "prims/jvmtiExport.hpp"
  45 #include "prims/methodHandles.hpp"
  46 #include "runtime/frame.inline.hpp"
  47 #include "runtime/sharedRuntime.hpp"
  48 #include "runtime/stubRoutines.hpp"
  49 #include "runtime/synchronizer.hpp"
  50 #include "utilities/powerOfTwo.hpp"
  51 
  52 #define __ Disassembler::hook<InterpreterMacroAssembler>(__FILE__, __LINE__, _masm)->
  53 
  54 // Address computation: local variables
  55 
  56 static inline Address iaddress(int n) {
  57   return Address(xlocals, Interpreter::local_offset_in_bytes(n));
  58 }
  59 
  60 static inline Address laddress(int n) {
  61   return iaddress(n + 1);
  62 }
  63 
  64 static inline Address faddress(int n) {
  65   return iaddress(n);
  66 }
  67 
  68 static inline Address daddress(int n) {
  69   return laddress(n);
  70 }
  71 
  72 static inline Address aaddress(int n) {
  73   return iaddress(n);
  74 }
  75 
  76 static inline Address iaddress(Register r, Register temp, InterpreterMacroAssembler* _masm) {
  77   _masm->shadd(temp, r, xlocals, temp, 3);
  78   return Address(temp, 0);
  79 }
  80 
  81 static inline Address laddress(Register r, Register temp, InterpreterMacroAssembler* _masm) {
  82   _masm->shadd(temp, r, xlocals, temp, 3);
  83   return Address(temp, Interpreter::local_offset_in_bytes(1));;
  84 }
  85 
  86 static inline Address faddress(Register r, Register temp, InterpreterMacroAssembler* _masm) {
  87   return iaddress(r, temp, _masm);
  88 }
  89 
  90 static inline Address daddress(Register r, Register temp, InterpreterMacroAssembler* _masm) {
  91   return laddress(r, temp, _masm);
  92 }
  93 
  94 static inline Address aaddress(Register r, Register temp, InterpreterMacroAssembler* _masm) {
  95   return iaddress(r, temp, _masm);
  96 }
  97 
  98 static inline Address at_rsp() {
  99   return Address(esp, 0);
 100 }
 101 
 102 // At top of Java expression stack which may be different than esp().  It
 103 // isn't for category 1 objects.
 104 static inline Address at_tos   () {
 105   return Address(esp,  Interpreter::expr_offset_in_bytes(0));
 106 }
 107 
 108 static inline Address at_tos_p1() {
 109   return Address(esp, Interpreter::expr_offset_in_bytes(1));
 110 }
 111 
 112 static inline Address at_tos_p2() {
 113   return Address(esp, Interpreter::expr_offset_in_bytes(2));
 114 }
 115 
 116 static inline Address at_tos_p3() {
 117   return Address(esp, Interpreter::expr_offset_in_bytes(3));
 118 }
 119 
 120 static inline Address at_tos_p4() {
 121   return Address(esp, Interpreter::expr_offset_in_bytes(4));
 122 }
 123 
 124 static inline Address at_tos_p5() {
 125   return Address(esp, Interpreter::expr_offset_in_bytes(5));
 126 }
 127 
 128 Address TemplateTable::at_bcp(int offset) {
 129   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 130   return Address(xbcp, offset);
 131 }
 132 
 133 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 134                                    Register temp_reg, bool load_bc_into_bc_reg /*=true*/,
 135                                    int byte_no) {
 136   if (!RewriteBytecodes) { return; }
 137   Label L_patch_done;
 138 
 139   switch (bc) {
 140     case Bytecodes::_fast_aputfield:  // fall through
 141     case Bytecodes::_fast_bputfield:  // fall through
 142     case Bytecodes::_fast_zputfield:  // fall through
 143     case Bytecodes::_fast_cputfield:  // fall through
 144     case Bytecodes::_fast_dputfield:  // fall through
 145     case Bytecodes::_fast_fputfield:  // fall through
 146     case Bytecodes::_fast_iputfield:  // fall through
 147     case Bytecodes::_fast_lputfield:  // fall through
 148     case Bytecodes::_fast_sputfield: {
 149       // We skip bytecode quickening for putfield instructions when
 150       // the put_code written to the constant pool cache is zero.
 151       // This is required so that every execution of this instruction
 152       // calls out to InterpreterRuntime::resolve_get_put to do
 153       // additional, required work.
 154       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 155       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 156       __ load_field_entry(temp_reg, bc_reg);
 157       if (byte_no == f1_byte) {
 158         __ la(temp_reg, Address(temp_reg, in_bytes(ResolvedFieldEntry::get_code_offset())));
 159       } else {
 160         __ la(temp_reg, Address(temp_reg, in_bytes(ResolvedFieldEntry::put_code_offset())));
 161       }
 162       // Load-acquire the bytecode to match store-release in ResolvedFieldEntry::fill_in()
 163       __ lbu(temp_reg, Address(temp_reg, 0));
 164       __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
 165       __ mv(bc_reg, bc);
 166       __ beqz(temp_reg, L_patch_done);
 167       break;
 168     }
 169     default:
 170       assert(byte_no == -1, "sanity");
 171       // the pair bytecodes have already done the load.
 172       if (load_bc_into_bc_reg) {
 173         __ mv(bc_reg, bc);
 174       }
 175   }
 176 
 177   if (JvmtiExport::can_post_breakpoint()) {
 178     Label L_fast_patch;
 179     // if a breakpoint is present we can't rewrite the stream directly
 180     __ load_unsigned_byte(temp_reg, at_bcp(0));
 181     __ subi(temp_reg, temp_reg, Bytecodes::_breakpoint); // temp_reg is temporary register.
 182     __ bnez(temp_reg, L_fast_patch);
 183     // Let breakpoint table handling rewrite to quicker bytecode
 184     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), xmethod, xbcp, bc_reg);
 185     __ j(L_patch_done);
 186     __ bind(L_fast_patch);
 187   }
 188 
 189 #ifdef ASSERT
 190   Label L_okay;
 191   __ load_unsigned_byte(temp_reg, at_bcp(0));
 192   __ beq(temp_reg, bc_reg, L_okay);
 193   __ subi(temp_reg, temp_reg, (int)Bytecodes::java_code(bc));
 194   __ beqz(temp_reg, L_okay);
 195   __ stop("patching the wrong bytecode");
 196   __ bind(L_okay);
 197 #endif
 198 
 199   // patch bytecode
 200   __ sb(bc_reg, at_bcp(0));
 201   __ bind(L_patch_done);
 202 }
 203 
 204 // Individual instructions
 205 
 206 void TemplateTable::nop() {
 207   transition(vtos, vtos);
 208   // nothing to do
 209 }
 210 
 211 void TemplateTable::shouldnotreachhere() {
 212   transition(vtos, vtos);
 213   __ stop("should not reach here bytecode");
 214 }
 215 
 216 void TemplateTable::aconst_null() {
 217   transition(vtos, atos);
 218   __ mv(x10, zr);
 219 }
 220 
 221 void TemplateTable::iconst(int value) {
 222   transition(vtos, itos);
 223   __ mv(x10, value);
 224 }
 225 
 226 void TemplateTable::lconst(int value) {
 227   transition(vtos, ltos);
 228   __ mv(x10, value);
 229 }
 230 
 231 void TemplateTable::fconst(int value) {
 232   transition(vtos, ftos);
 233   static float fBuf[2] = {1.0, 2.0};
 234   __ mv(t0, (intptr_t)fBuf);
 235   switch (value) {
 236     case 0:
 237       __ fmv_w_x(f10, zr);
 238       break;
 239     case 1:
 240       __ flw(f10, Address(t0, 0));
 241       break;
 242     case 2:
 243       __ flw(f10, Address(t0, sizeof(float)));
 244       break;
 245     default:
 246       ShouldNotReachHere();
 247   }
 248 }
 249 
 250 void TemplateTable::dconst(int value) {
 251   transition(vtos, dtos);
 252   static double dBuf[2] = {1.0, 2.0};
 253   __ mv(t0, (intptr_t)dBuf);
 254   switch (value) {
 255     case 0:
 256       __ fmv_d_x(f10, zr);
 257       break;
 258     case 1:
 259       __ fld(f10, Address(t0, 0));
 260       break;
 261     case 2:
 262       __ fld(f10, Address(t0, sizeof(double)));
 263       break;
 264     default:
 265       ShouldNotReachHere();
 266   }
 267 }
 268 
 269 void TemplateTable::bipush() {
 270   transition(vtos, itos);
 271   __ load_signed_byte(x10, at_bcp(1));
 272 }
 273 
 274 void TemplateTable::sipush() {
 275   transition(vtos, itos);
 276   __ load_signed_byte(x10, at_bcp(1));
 277   __ load_unsigned_byte(t1, at_bcp(2));
 278   __ slli(x10, x10, 8);
 279   __ add(x10, x10, t1);
 280 }
 281 
 282 void TemplateTable::ldc(LdcType type) {
 283   transition(vtos, vtos);
 284   Label call_ldc, notFloat, notClass, notInt, Done;
 285 
 286   if (is_ldc_wide(type)) {
 287    __ get_unsigned_2_byte_index_at_bcp(x11, 1);
 288   } else {
 289    __ load_unsigned_byte(x11, at_bcp(1));
 290   }
 291   __ get_cpool_and_tags(x12, x10);
 292 
 293   const int base_offset = ConstantPool::header_size() * wordSize;
 294   const int tags_offset = Array<u1>::base_offset_in_bytes();
 295 
 296   // get type
 297   __ addi(x13, x11, tags_offset);
 298   __ add(x13, x10, x13);
 299   __ lbu(x13, Address(x13, 0));
 300   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
 301 
 302   // unresolved class - get the resolved class
 303   __ mv(t1, (u1)JVM_CONSTANT_UnresolvedClass);
 304   __ beq(x13, t1, call_ldc);
 305 
 306   // unresolved class in error state - call into runtime to throw the error
 307   // from the first resolution attempt
 308   __ mv(t1, (u1)JVM_CONSTANT_UnresolvedClassInError);
 309   __ beq(x13, t1, call_ldc);
 310 
 311   // resolved class - need to call vm to get java mirror of the class
 312   __ mv(t1, (u1)JVM_CONSTANT_Class);
 313   __ bne(x13, t1, notClass);
 314 
 315   __ bind(call_ldc);
 316   __ mv(c_rarg1, is_ldc_wide(type) ? 1 : 0);
 317   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 318   __ push_ptr(x10);
 319   __ verify_oop(x10);
 320   __ j(Done);
 321 
 322   __ bind(notClass);
 323   __ mv(t1, (u1)JVM_CONSTANT_Float);
 324   __ bne(x13, t1, notFloat);
 325 
 326   // ftos
 327   __ shadd(x11, x11, x12, x11, 3);
 328   __ flw(f10, Address(x11, base_offset));
 329   __ push_f(f10);
 330   __ j(Done);
 331 
 332   __ bind(notFloat);
 333 
 334   __ mv(t1, (u1)JVM_CONSTANT_Integer);
 335   __ bne(x13, t1, notInt);
 336 
 337   // itos
 338   __ shadd(x11, x11, x12, x11, 3);
 339   __ lw(x10, Address(x11, base_offset));
 340   __ push_i(x10);
 341   __ j(Done);
 342 
 343   __ bind(notInt);
 344   condy_helper(Done);
 345 
 346   __ bind(Done);
 347 }
 348 
 349 // Fast path for caching oop constants.
 350 void TemplateTable::fast_aldc(LdcType type) {
 351   transition(vtos, atos);
 352 
 353   const Register result = x10;
 354   const Register tmp = x11;
 355   const Register rarg = x12;
 356 
 357   const int index_size = is_ldc_wide(type) ? sizeof(u2) : sizeof(u1);
 358 
 359   Label resolved;
 360 
 361   // We are resolved if the resolved reference cache entry contains a
 362   // non-null object (String, MethodType, etc.)
 363   assert_different_registers(result, tmp);
 364   // register result is trashed by next load, let's use it as temporary register
 365   __ get_cache_index_at_bcp(tmp, result, 1, index_size);
 366   __ load_resolved_reference_at_index(result, tmp);
 367   __ bnez(result, resolved);
 368 
 369   const address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 370 
 371   // first time invocation - must resolve first
 372   __ mv(rarg, (int)bytecode());
 373   __ call_VM(result, entry, rarg);
 374 
 375   __ bind(resolved);
 376 
 377   { // Check for the null sentinel.
 378     // If we just called the VM, it already did the mapping for us,
 379     // but it's harmless to retry.
 380     Label notNull;
 381 
 382     // Stash null_sentinel address to get its value later
 383     int32_t offset = 0;
 384     __ mv(rarg, Universe::the_null_sentinel_addr(), offset);
 385     __ ld(tmp, Address(rarg, offset));
 386     __ resolve_oop_handle(tmp, x15, t1);
 387     __ bne(result, tmp, notNull);
 388     __ mv(result, zr);  // null object reference
 389     __ bind(notNull);
 390   }
 391 
 392   if (VerifyOops) {
 393     // Safe to call with 0 result
 394     __ verify_oop(result);
 395   }
 396 }
 397 
 398 void TemplateTable::ldc2_w() {
 399     transition(vtos, vtos);
 400     Label notDouble, notLong, Done;
 401     __ get_unsigned_2_byte_index_at_bcp(x10, 1);
 402 
 403     __ get_cpool_and_tags(x11, x12);
 404     const int base_offset = ConstantPool::header_size() * wordSize;
 405     const int tags_offset = Array<u1>::base_offset_in_bytes();
 406 
 407     // get type
 408     __ add(x12, x12, x10);
 409     __ load_unsigned_byte(x12, Address(x12, tags_offset));
 410     __ mv(t1, JVM_CONSTANT_Double);
 411     __ bne(x12, t1, notDouble);
 412 
 413     // dtos
 414     __ shadd(x12, x10, x11, x12, 3);
 415     __ fld(f10, Address(x12, base_offset));
 416     __ push_d(f10);
 417     __ j(Done);
 418 
 419     __ bind(notDouble);
 420     __ mv(t1, (int)JVM_CONSTANT_Long);
 421     __ bne(x12, t1, notLong);
 422 
 423     // ltos
 424     __ shadd(x10, x10, x11, x10, 3);
 425     __ ld(x10, Address(x10, base_offset));
 426     __ push_l(x10);
 427     __ j(Done);
 428 
 429     __ bind(notLong);
 430     condy_helper(Done);
 431     __ bind(Done);
 432 }
 433 
 434 void TemplateTable::condy_helper(Label& Done) {
 435   const Register obj = x10;
 436   const Register rarg = x11;
 437   const Register flags = x12;
 438   const Register off = x13;
 439 
 440   const address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 441 
 442   __ mv(rarg, (int) bytecode());
 443   __ call_VM(obj, entry, rarg);
 444 
 445   __ get_vm_result_metadata(flags, xthread);
 446 
 447   // VMr = obj = base address to find primitive value to push
 448   // VMr2 = flags = (tos, off) using format of CPCE::_flags
 449   __ mv(off, flags);
 450   __ mv(t0, ConstantPoolCache::field_index_mask);
 451   __ andrw(off, off, t0);
 452 
 453   __ add(off, obj, off);
 454   const Address field(off, 0); // base + R---->base + offset
 455 
 456   __ slli(flags, flags, XLEN - (ConstantPoolCache::tos_state_shift + ConstantPoolCache::tos_state_bits));
 457   __ srli(flags, flags, XLEN - ConstantPoolCache::tos_state_bits); // (1 << 5) - 4 --> 28~31==> flags:0~3
 458 
 459   switch (bytecode()) {
 460     case Bytecodes::_ldc:   // fall through
 461     case Bytecodes::_ldc_w: {
 462       // tos in (itos, ftos, stos, btos, ctos, ztos)
 463       Label notInt, notFloat, notShort, notByte, notChar, notBool;
 464       __ mv(t1, itos);
 465       __ bne(flags, t1, notInt);
 466       // itos
 467       __ lw(x10, field);
 468       __ push(itos);
 469       __ j(Done);
 470 
 471       __ bind(notInt);
 472       __ mv(t1, ftos);
 473       __ bne(flags, t1, notFloat);
 474       // ftos
 475       __ load_float(field);
 476       __ push(ftos);
 477       __ j(Done);
 478 
 479       __ bind(notFloat);
 480       __ mv(t1, stos);
 481       __ bne(flags, t1, notShort);
 482       // stos
 483       __ load_signed_short(x10, field);
 484       __ push(stos);
 485       __ j(Done);
 486 
 487       __ bind(notShort);
 488       __ mv(t1, btos);
 489       __ bne(flags, t1, notByte);
 490       // btos
 491       __ load_signed_byte(x10, field);
 492       __ push(btos);
 493       __ j(Done);
 494 
 495       __ bind(notByte);
 496       __ mv(t1, ctos);
 497       __ bne(flags, t1, notChar);
 498       // ctos
 499       __ load_unsigned_short(x10, field);
 500       __ push(ctos);
 501       __ j(Done);
 502 
 503       __ bind(notChar);
 504       __ mv(t1, ztos);
 505       __ bne(flags, t1, notBool);
 506       // ztos
 507       __ load_signed_byte(x10, field);
 508       __ push(ztos);
 509       __ j(Done);
 510 
 511       __ bind(notBool);
 512       break;
 513     }
 514 
 515     case Bytecodes::_ldc2_w: {
 516       Label notLong, notDouble;
 517       __ mv(t1, ltos);
 518       __ bne(flags, t1, notLong);
 519       // ltos
 520       __ ld(x10, field);
 521       __ push(ltos);
 522       __ j(Done);
 523 
 524       __ bind(notLong);
 525       __ mv(t1, dtos);
 526       __ bne(flags, t1, notDouble);
 527       // dtos
 528       __ load_double(field);
 529       __ push(dtos);
 530       __ j(Done);
 531 
 532       __ bind(notDouble);
 533       break;
 534     }
 535 
 536     default:
 537       ShouldNotReachHere();
 538   }
 539 
 540   __ stop("bad ldc/condy");
 541 }
 542 
 543 void TemplateTable::locals_index(Register reg, int offset) {
 544   __ lbu(reg, at_bcp(offset));
 545   __ neg(reg, reg);
 546 }
 547 
 548 void TemplateTable::iload() {
 549   iload_internal();
 550 }
 551 
 552 void TemplateTable::nofast_iload() {
 553   iload_internal(may_not_rewrite);
 554 }
 555 
 556 void TemplateTable::iload_internal(RewriteControl rc) {
 557   transition(vtos, itos);
 558   if (RewriteFrequentPairs && rc == may_rewrite) {
 559     Label rewrite, done;
 560     const Register bc = x14;
 561 
 562     // get next bytecode
 563     __ load_unsigned_byte(x11, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 564 
 565     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 566     // last two iloads in a pair.  Comparing against fast_iload means that
 567     // the next bytecode is neither an iload or a caload, and therefore
 568     // an iload pair.
 569     __ mv(t1, Bytecodes::_iload);
 570     __ beq(x11, t1, done);
 571 
 572     // if _fast_iload rewrite to _fast_iload2
 573     __ mv(t1, Bytecodes::_fast_iload);
 574     __ mv(bc, Bytecodes::_fast_iload2);
 575     __ beq(x11, t1, rewrite);
 576 
 577     // if _caload rewrite to _fast_icaload
 578     __ mv(t1, Bytecodes::_caload);
 579     __ mv(bc, Bytecodes::_fast_icaload);
 580     __ beq(x11, t1, rewrite);
 581 
 582     // else rewrite to _fast_iload
 583     __ mv(bc, Bytecodes::_fast_iload);
 584 
 585     // rewrite
 586     // bc: new bytecode
 587     __ bind(rewrite);
 588     patch_bytecode(Bytecodes::_iload, bc, x11, false);
 589     __ bind(done);
 590 
 591   }
 592 
 593   // do iload, get the local value into tos
 594   locals_index(x11);
 595   __ lw(x10, iaddress(x11, x10, _masm));
 596 }
 597 
 598 void TemplateTable::fast_iload2() {
 599   transition(vtos, itos);
 600   locals_index(x11);
 601   __ lw(x10, iaddress(x11, x10, _masm));
 602   __ push(itos);
 603   locals_index(x11, 3);
 604   __ lw(x10, iaddress(x11, x10, _masm));
 605 }
 606 
 607 void TemplateTable::fast_iload() {
 608   transition(vtos, itos);
 609   locals_index(x11);
 610   __ lw(x10, iaddress(x11, x10, _masm));
 611 }
 612 
 613 void TemplateTable::lload() {
 614   transition(vtos, ltos);
 615   __ lbu(x11, at_bcp(1));
 616   __ slli(x11, x11, LogBytesPerWord);
 617   __ sub(x11, xlocals, x11);
 618   __ ld(x10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 619 }
 620 
 621 void TemplateTable::fload() {
 622   transition(vtos, ftos);
 623   locals_index(x11);
 624   __ flw(f10, faddress(x11, t0, _masm));
 625 }
 626 
 627 void TemplateTable::dload() {
 628   transition(vtos, dtos);
 629   __ lbu(x11, at_bcp(1));
 630   __ slli(x11, x11, LogBytesPerWord);
 631   __ sub(x11, xlocals, x11);
 632   __ fld(f10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 633 }
 634 
 635 void TemplateTable::aload() {
 636   transition(vtos, atos);
 637   locals_index(x11);
 638   __ ld(x10, iaddress(x11, x10, _masm));
 639 }
 640 
 641 void TemplateTable::locals_index_wide(Register reg) {
 642   assert_different_registers(reg, t1);
 643   // Convert the 16-bit value into native byte-ordering and zero-extend
 644   __ lbu(reg, at_bcp(2));
 645   __ lbu(t1, at_bcp(3));
 646   __ slli(reg, reg, 8);
 647   __ orr(reg, reg, t1);
 648   __ neg(reg, reg);
 649 }
 650 
 651 void TemplateTable::wide_iload() {
 652   transition(vtos, itos);
 653   locals_index_wide(x11);
 654   __ lw(x10, iaddress(x11, t0, _masm));
 655 }
 656 
 657 void TemplateTable::wide_lload() {
 658   transition(vtos, ltos);
 659   // Convert the 16-bit value into native byte-ordering and zero-extend
 660   __ lbu(x11, at_bcp(2));
 661   __ lbu(t1, at_bcp(3));
 662   __ slli(x11, x11, 8);
 663   __ orr(x11, x11, t1);
 664 
 665   __ slli(x11, x11, LogBytesPerWord);
 666   __ sub(x11, xlocals, x11);
 667   __ ld(x10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 668 }
 669 
 670 void TemplateTable::wide_fload() {
 671   transition(vtos, ftos);
 672   locals_index_wide(x11);
 673   __ flw(f10, faddress(x11, t0, _masm));
 674 }
 675 
 676 void TemplateTable::wide_dload() {
 677   transition(vtos, dtos);
 678   // Convert the 16-bit value into native byte-ordering and zero-extend
 679   __ lbu(x11, at_bcp(2));
 680   __ lbu(t1, at_bcp(3));
 681   __ slli(x11, x11, 8);
 682   __ orr(x11, x11, t1);
 683 
 684   __ slli(x11, x11, LogBytesPerWord);
 685   __ sub(x11, xlocals, x11);
 686   __ fld(f10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 687 }
 688 
 689 void TemplateTable::wide_aload() {
 690   transition(vtos, atos);
 691   locals_index_wide(x11);
 692   __ ld(x10, aaddress(x11, t0, _masm));
 693 }
 694 
 695 void TemplateTable::index_check(Register array, Register index) {
 696   // destroys x11, t0, t1
 697   // sign extend index for use by indexed load
 698   // check index
 699   const Register length = t0;
 700   __ lwu(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 701   if (index != x11) {
 702     assert(x11 != array, "different registers");
 703     __ mv(x11, index);
 704   }
 705   Label ok;
 706   __ sext(index, index, 32);
 707   __ bltu(index, length, ok);
 708   __ mv(x13, array);
 709   __ mv(t1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 710   __ jr(t1);
 711   __ bind(ok);
 712 }
 713 
 714 void TemplateTable::iaload() {
 715   transition(itos, itos);
 716   __ mv(x11, x10);
 717   __ pop_ptr(x10);
 718   // x10: array
 719   // x11: index
 720   index_check(x10, x11); // leaves index in x11
 721   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
 722   __ shadd(x10, x11, x10, t0, 2);
 723   __ access_load_at(T_INT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 724   __ sext(x10, x10, 32);
 725 }
 726 
 727 void TemplateTable::laload() {
 728   transition(itos, ltos);
 729   __ mv(x11, x10);
 730   __ pop_ptr(x10);
 731   // x10: array
 732   // x11: index
 733   index_check(x10, x11); // leaves index in x11
 734   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
 735   __ shadd(x10, x11, x10, t0, 3);
 736   __ access_load_at(T_LONG, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 737 }
 738 
 739 void TemplateTable::faload() {
 740   transition(itos, ftos);
 741   __ mv(x11, x10);
 742   __ pop_ptr(x10);
 743   // x10: array
 744   // x11: index
 745   index_check(x10, x11); // leaves index in x11
 746   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
 747   __ shadd(x10, x11, x10, t0, 2);
 748   __ access_load_at(T_FLOAT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 749 }
 750 
 751 void TemplateTable::daload() {
 752   transition(itos, dtos);
 753   __ mv(x11, x10);
 754   __ pop_ptr(x10);
 755   // x10: array
 756   // x11: index
 757   index_check(x10, x11); // leaves index in x11
 758   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
 759   __ shadd(x10, x11, x10, t0, 3);
 760   __ access_load_at(T_DOUBLE, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 761 }
 762 
 763 void TemplateTable::aaload() {
 764   transition(itos, atos);
 765   __ mv(x11, x10);
 766   __ pop_ptr(x10);
 767   // x10: array
 768   // x11: index
 769   index_check(x10, x11); // leaves index in x11
 770   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 771   __ shadd(x10, x11, x10, t0, LogBytesPerHeapOop);
 772   __ load_heap_oop(x10, Address(x10), x28, x29, IS_ARRAY);
 773 }
 774 
 775 void TemplateTable::baload() {
 776   transition(itos, itos);
 777   __ mv(x11, x10);
 778   __ pop_ptr(x10);
 779   // x10: array
 780   // x11: index
 781   index_check(x10, x11); // leaves index in x11
 782   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
 783   __ shadd(x10, x11, x10, t0, 0);
 784   __ access_load_at(T_BYTE, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 785 }
 786 
 787 void TemplateTable::caload() {
 788   transition(itos, itos);
 789   __ mv(x11, x10);
 790   __ pop_ptr(x10);
 791   // x10: array
 792   // x11: index
 793   index_check(x10, x11); // leaves index in x11
 794   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
 795   __ shadd(x10, x11, x10, t0, 1);
 796   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 797 }
 798 
 799 // iload followed by caload frequent pair
 800 void TemplateTable::fast_icaload() {
 801   transition(vtos, itos);
 802   // load index out of locals
 803   locals_index(x12);
 804   __ lw(x11, iaddress(x12, x11, _masm));
 805   __ pop_ptr(x10);
 806 
 807   // x10: array
 808   // x11: index
 809   index_check(x10, x11); // leaves index in x11, kills t0
 810   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1); // addi, max imm is 2^11
 811   __ shadd(x10, x11, x10, t0, 1);
 812   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 813 }
 814 
 815 void TemplateTable::saload() {
 816   transition(itos, itos);
 817   __ mv(x11, x10);
 818   __ pop_ptr(x10);
 819   // x10: array
 820   // x11: index
 821   index_check(x10, x11); // leaves index in x11, kills t0
 822   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_SHORT) >> 1);
 823   __ shadd(x10, x11, x10, t0, 1);
 824   __ access_load_at(T_SHORT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 825 }
 826 
 827 void TemplateTable::iload(int n) {
 828   transition(vtos, itos);
 829   __ lw(x10, iaddress(n));
 830 }
 831 
 832 void TemplateTable::lload(int n) {
 833   transition(vtos, ltos);
 834   __ ld(x10, laddress(n));
 835 }
 836 
 837 void TemplateTable::fload(int n) {
 838   transition(vtos, ftos);
 839   __ flw(f10, faddress(n));
 840 }
 841 
 842 void TemplateTable::dload(int n) {
 843   transition(vtos, dtos);
 844   __ fld(f10, daddress(n));
 845 }
 846 
 847 void TemplateTable::aload(int n) {
 848   transition(vtos, atos);
 849   __ ld(x10, iaddress(n));
 850 }
 851 
 852 void TemplateTable::aload_0() {
 853   aload_0_internal();
 854 }
 855 
 856 void TemplateTable::nofast_aload_0() {
 857   aload_0_internal(may_not_rewrite);
 858 }
 859 
 860 void TemplateTable::aload_0_internal(RewriteControl rc) {
 861   // According to bytecode histograms, the pairs:
 862   //
 863   // _aload_0, _fast_igetfield
 864   // _aload_0, _fast_agetfield
 865   // _aload_0, _fast_fgetfield
 866   //
 867   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 868   // _aload_0 bytecode checks if the next bytecode is either
 869   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 870   // rewrites the current bytecode into a pair bytecode; otherwise it
 871   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 872   // the pair check anymore.
 873   //
 874   // Note: If the next bytecode is _getfield, the rewrite must be
 875   //       delayed, otherwise we may miss an opportunity for a pair.
 876   //
 877   // Also rewrite frequent pairs
 878   //   aload_0, aload_1
 879   //   aload_0, iload_1
 880   // These bytecodes with a small amount of code are most profitable
 881   // to rewrite
 882   if (RewriteFrequentPairs && rc == may_rewrite) {
 883     Label rewrite, done;
 884     const Register bc = x14;
 885 
 886     // get next bytecode
 887     __ load_unsigned_byte(x11, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 888 
 889     // if _getfield then wait with rewrite
 890     __ mv(t1, Bytecodes::Bytecodes::_getfield);
 891     __ beq(x11, t1, done);
 892 
 893     // if _igetfield then rewrite to _fast_iaccess_0
 894     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 895     __ mv(t1, Bytecodes::_fast_igetfield);
 896     __ mv(bc, Bytecodes::_fast_iaccess_0);
 897     __ beq(x11, t1, rewrite);
 898 
 899     // if _agetfield then rewrite to _fast_aaccess_0
 900     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 901     __ mv(t1, Bytecodes::_fast_agetfield);
 902     __ mv(bc, Bytecodes::_fast_aaccess_0);
 903     __ beq(x11, t1, rewrite);
 904 
 905     // if _fgetfield then rewrite to _fast_faccess_0
 906     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 907     __ mv(t1, Bytecodes::_fast_fgetfield);
 908     __ mv(bc, Bytecodes::_fast_faccess_0);
 909     __ beq(x11, t1, rewrite);
 910 
 911     // else rewrite to _fast_aload0
 912     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 913     __ mv(bc, Bytecodes::Bytecodes::_fast_aload_0);
 914 
 915     // rewrite
 916     // bc: new bytecode
 917     __ bind(rewrite);
 918     patch_bytecode(Bytecodes::_aload_0, bc, x11, false);
 919 
 920     __ bind(done);
 921   }
 922 
 923   // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
 924   aload(0);
 925 }
 926 
 927 void TemplateTable::istore() {
 928   transition(itos, vtos);
 929   locals_index(x11);
 930   __ sw(x10, iaddress(x11, t0, _masm));
 931 }
 932 
 933 void TemplateTable::lstore() {
 934   transition(ltos, vtos);
 935   locals_index(x11);
 936   __ sd(x10, laddress(x11, t0, _masm));
 937 }
 938 
 939 void TemplateTable::fstore() {
 940   transition(ftos, vtos);
 941   locals_index(x11);
 942   __ fsw(f10, iaddress(x11, t0, _masm));
 943 }
 944 
 945 void TemplateTable::dstore() {
 946   transition(dtos, vtos);
 947   locals_index(x11);
 948   __ fsd(f10, daddress(x11, t0, _masm));
 949 }
 950 
 951 void TemplateTable::astore() {
 952   transition(vtos, vtos);
 953   __ pop_ptr(x10);
 954   locals_index(x11);
 955   __ sd(x10, aaddress(x11, t0, _masm));
 956 }
 957 
 958 void TemplateTable::wide_istore() {
 959   transition(vtos, vtos);
 960   __ pop_i();
 961   locals_index_wide(x11);
 962   __ sw(x10, iaddress(x11, t0, _masm));
 963 }
 964 
 965 void TemplateTable::wide_lstore() {
 966   transition(vtos, vtos);
 967   __ pop_l();
 968   locals_index_wide(x11);
 969   __ sd(x10, laddress(x11, t0, _masm));
 970 }
 971 
 972 void TemplateTable::wide_fstore() {
 973   transition(vtos, vtos);
 974   __ pop_f();
 975   locals_index_wide(x11);
 976   __ fsw(f10, faddress(x11, t0, _masm));
 977 }
 978 
 979 void TemplateTable::wide_dstore() {
 980   transition(vtos, vtos);
 981   __ pop_d();
 982   locals_index_wide(x11);
 983   __ fsd(f10, daddress(x11, t0, _masm));
 984 }
 985 
 986 void TemplateTable::wide_astore() {
 987   transition(vtos, vtos);
 988   __ pop_ptr(x10);
 989   locals_index_wide(x11);
 990   __ sd(x10, aaddress(x11, t0, _masm));
 991 }
 992 
 993 void TemplateTable::iastore() {
 994   transition(itos, vtos);
 995   __ pop_i(x11);
 996   __ pop_ptr(x13);
 997   // x10: value
 998   // x11: index
 999   // x13: array
1000   index_check(x13, x11); // prefer index in x11
1001   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
1002   __ shadd(t0, x11, x13, t0, 2);
1003   __ access_store_at(T_INT, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg, noreg);
1004 }
1005 
1006 void TemplateTable::lastore() {
1007   transition(ltos, vtos);
1008   __ pop_i(x11);
1009   __ pop_ptr(x13);
1010   // x10: value
1011   // x11: index
1012   // x13: array
1013   index_check(x13, x11); // prefer index in x11
1014   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
1015   __ shadd(t0, x11, x13, t0, 3);
1016   __ access_store_at(T_LONG, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg, noreg);
1017 }
1018 
1019 void TemplateTable::fastore() {
1020   transition(ftos, vtos);
1021   __ pop_i(x11);
1022   __ pop_ptr(x13);
1023   // f10: value
1024   // x11:  index
1025   // x13:  array
1026   index_check(x13, x11); // prefer index in x11
1027   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
1028   __ shadd(t0, x11, x13, t0, 2);
1029   __ access_store_at(T_FLOAT, IN_HEAP | IS_ARRAY, Address(t0, 0), noreg /* ftos */, noreg, noreg, noreg);
1030 }
1031 
1032 void TemplateTable::dastore() {
1033   transition(dtos, vtos);
1034   __ pop_i(x11);
1035   __ pop_ptr(x13);
1036   // f10: value
1037   // x11:  index
1038   // x13:  array
1039   index_check(x13, x11); // prefer index in x11
1040   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
1041   __ shadd(t0, x11, x13, t0, 3);
1042   __ access_store_at(T_DOUBLE, IN_HEAP | IS_ARRAY, Address(t0, 0), noreg /* dtos */, noreg, noreg, noreg);
1043 }
1044 
1045 void TemplateTable::aastore() {
1046   Label is_null, ok_is_subtype, done;
1047   transition(vtos, vtos);
1048   // stack: ..., array, index, value
1049   __ ld(x10, at_tos());    // value
1050   __ ld(x12, at_tos_p1()); // index
1051   __ ld(x13, at_tos_p2()); // array
1052 
1053   index_check(x13, x12);     // kills x11
1054   __ addi(x14, x12, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
1055   __ shadd(x14, x14, x13, x14, LogBytesPerHeapOop);
1056 
1057   Address element_address(x14, 0);
1058 
1059   // do array store check - check for null value first
1060   __ beqz(x10, is_null);
1061 
1062   // Move subklass into x11
1063   __ load_klass(x11, x10);
1064   // Move superklass into x10
1065   __ load_klass(x10, x13);
1066   __ ld(x10, Address(x10,
1067                      ObjArrayKlass::element_klass_offset()));
1068   // Compress array + index * oopSize + 12 into a single register.  Frees x12.
1069 
1070   // Generate subtype check.  Blows x12, x15
1071   // Superklass in x10.  Subklass in x11.
1072   __ gen_subtype_check(x11, ok_is_subtype);
1073 
1074   // Come here on failure
1075   // object is at TOS
1076   __ j(RuntimeAddress(Interpreter::_throw_ArrayStoreException_entry));
1077 
1078   // Come here on success
1079   __ bind(ok_is_subtype);
1080 
1081   // Get the value we will store
1082   __ ld(x10, at_tos());
1083   // Now store using the appropriate barrier
1084   __ store_heap_oop(element_address, x10, x28, x29, x13, IS_ARRAY);
1085   __ j(done);
1086 
1087   // Have a null in x10, x13=array, x12=index.  Store null at ary[idx]
1088   __ bind(is_null);
1089   __ profile_null_seen(x12);
1090 
1091   // Store a null
1092   __ store_heap_oop(element_address, noreg, x28, x29, x13, IS_ARRAY);
1093 
1094   // Pop stack arguments
1095   __ bind(done);
1096   __ addi(esp, esp, 3 * Interpreter::stackElementSize);
1097 }
1098 
1099 void TemplateTable::bastore() {
1100   transition(itos, vtos);
1101   __ pop_i(x11);
1102   __ pop_ptr(x13);
1103   // x10: value
1104   // x11: index
1105   // x13: array
1106   index_check(x13, x11); // prefer index in x11
1107 
1108   // Need to check whether array is boolean or byte
1109   // since both types share the bastore bytecode.
1110   __ load_klass(x12, x13);
1111   __ lwu(x12, Address(x12, Klass::layout_helper_offset()));
1112   Label L_skip;
1113   __ test_bit(t0, x12, exact_log2(Klass::layout_helper_boolean_diffbit()));
1114   __ beqz(t0, L_skip);
1115   __ andi(x10, x10, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1116   __ bind(L_skip);
1117 
1118   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
1119 
1120   __ add(x11, x13, x11);
1121   __ access_store_at(T_BYTE, IN_HEAP | IS_ARRAY, Address(x11, 0), x10, noreg, noreg, noreg);
1122 }
1123 
1124 void TemplateTable::castore() {
1125   transition(itos, vtos);
1126   __ pop_i(x11);
1127   __ pop_ptr(x13);
1128   // x10: value
1129   // x11: index
1130   // x13: array
1131   index_check(x13, x11); // prefer index in x11
1132   __ addi(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
1133   __ shadd(t0, x11, x13, t0, 1);
1134   __ access_store_at(T_CHAR, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg, noreg);
1135 }
1136 
1137 void TemplateTable::sastore() {
1138   castore();
1139 }
1140 
1141 void TemplateTable::istore(int n) {
1142   transition(itos, vtos);
1143   __ sd(x10, iaddress(n));
1144 }
1145 
1146 void TemplateTable::lstore(int n) {
1147   transition(ltos, vtos);
1148   __ sd(x10, laddress(n));
1149 }
1150 
1151 void TemplateTable::fstore(int n) {
1152   transition(ftos, vtos);
1153   __ fsw(f10, faddress(n));
1154 }
1155 
1156 void TemplateTable::dstore(int n) {
1157   transition(dtos, vtos);
1158   __ fsd(f10, daddress(n));
1159 }
1160 
1161 void TemplateTable::astore(int n) {
1162   transition(vtos, vtos);
1163   __ pop_ptr(x10);
1164   __ sd(x10, iaddress(n));
1165 }
1166 
1167 void TemplateTable::pop() {
1168   transition(vtos, vtos);
1169   __ addi(esp, esp, Interpreter::stackElementSize);
1170 }
1171 
1172 void TemplateTable::pop2() {
1173   transition(vtos, vtos);
1174   __ addi(esp, esp, 2 * Interpreter::stackElementSize);
1175 }
1176 
1177 void TemplateTable::dup() {
1178   transition(vtos, vtos);
1179   __ ld(x10, Address(esp, 0));
1180   __ push_reg(x10);
1181   // stack: ..., a, a
1182 }
1183 
1184 void TemplateTable::dup_x1() {
1185   transition(vtos, vtos);
1186   // stack: ..., a, b
1187   __ ld(x10, at_tos());  // load b
1188   __ ld(x12, at_tos_p1());  // load a
1189   __ sd(x10, at_tos_p1());  // store b
1190   __ sd(x12, at_tos());  // store a
1191   __ push_reg(x10);                  // push b
1192   // stack: ..., b, a, b
1193 }
1194 
1195 void TemplateTable::dup_x2() {
1196   transition(vtos, vtos);
1197   // stack: ..., a, b, c
1198   __ ld(x10, at_tos());  // load c
1199   __ ld(x12, at_tos_p2());  // load a
1200   __ sd(x10, at_tos_p2());  // store c in a
1201   __ push_reg(x10);      // push c
1202   // stack: ..., c, b, c, c
1203   __ ld(x10, at_tos_p2());  // load b
1204   __ sd(x12, at_tos_p2());  // store a in b
1205   // stack: ..., c, a, c, c
1206   __ sd(x10, at_tos_p1());  // store b in c
1207   // stack: ..., c, a, b, c
1208 }
1209 
1210 void TemplateTable::dup2() {
1211   transition(vtos, vtos);
1212   // stack: ..., a, b
1213   __ ld(x10, at_tos_p1());  // load a
1214   __ push_reg(x10);                  // push a
1215   __ ld(x10, at_tos_p1());  // load b
1216   __ push_reg(x10);                  // push b
1217   // stack: ..., a, b, a, b
1218 }
1219 
1220 void TemplateTable::dup2_x1() {
1221   transition(vtos, vtos);
1222   // stack: ..., a, b, c
1223   __ ld(x12, at_tos());     // load c
1224   __ ld(x10, at_tos_p1());  // load b
1225   __ push_reg(x10);             // push b
1226   __ push_reg(x12);             // push c
1227   // stack: ..., a, b, c, b, c
1228   __ sd(x12, at_tos_p3());  // store c in b
1229   // stack: ..., a, c, c, b, c
1230   __ ld(x12, at_tos_p4());  // load a
1231   __ sd(x12, at_tos_p2());  // store a in 2nd c
1232   // stack: ..., a, c, a, b, c
1233   __ sd(x10, at_tos_p4());  // store b in a
1234   // stack: ..., b, c, a, b, c
1235 }
1236 
1237 void TemplateTable::dup2_x2() {
1238   transition(vtos, vtos);
1239   // stack: ..., a, b, c, d
1240   __ ld(x12, at_tos());     // load d
1241   __ ld(x10, at_tos_p1());  // load c
1242   __ push_reg(x10);             // push c
1243   __ push_reg(x12);             // push d
1244   // stack: ..., a, b, c, d, c, d
1245   __ ld(x10, at_tos_p4());  // load b
1246   __ sd(x10, at_tos_p2());  // store b in d
1247   __ sd(x12, at_tos_p4());  // store d in b
1248   // stack: ..., a, d, c, b, c, d
1249   __ ld(x12, at_tos_p5());  // load a
1250   __ ld(x10, at_tos_p3());  // load c
1251   __ sd(x12, at_tos_p3());  // store a in c
1252   __ sd(x10, at_tos_p5());  // store c in a
1253   // stack: ..., c, d, a, b, c, d
1254 }
1255 
1256 void TemplateTable::swap() {
1257   transition(vtos, vtos);
1258   // stack: ..., a, b
1259   __ ld(x12, at_tos_p1());  // load a
1260   __ ld(x10, at_tos());     // load b
1261   __ sd(x12, at_tos());     // store a in b
1262   __ sd(x10, at_tos_p1());  // store b in a
1263   // stack: ..., b, a
1264 }
1265 
1266 void TemplateTable::iop2(Operation op) {
1267   transition(itos, itos);
1268   // x10 <== x11 op x10
1269   __ pop_i(x11);
1270   switch (op) {
1271     case add  : __ addw(x10, x11, x10);  break;
1272     case sub  : __ subw(x10, x11, x10);  break;
1273     case mul  : __ mulw(x10, x11, x10);  break;
1274     case _and : __ andrw(x10, x11, x10); break;
1275     case _or  : __ orrw(x10, x11, x10);  break;
1276     case _xor : __ xorrw(x10, x11, x10); break;
1277     case shl  : __ sllw(x10, x11, x10);  break;
1278     case shr  : __ sraw(x10, x11, x10);  break;
1279     case ushr : __ srlw(x10, x11, x10);  break;
1280     default   : ShouldNotReachHere();
1281   }
1282 }
1283 
1284 void TemplateTable::lop2(Operation op) {
1285   transition(ltos, ltos);
1286   // x10 <== x11 op x10
1287   __ pop_l(x11);
1288   switch (op) {
1289     case add  : __ add(x10, x11, x10);  break;
1290     case sub  : __ sub(x10, x11, x10);  break;
1291     case mul  : __ mul(x10, x11, x10);  break;
1292     case _and : __ andr(x10, x11, x10); break;
1293     case _or  : __ orr(x10, x11, x10);  break;
1294     case _xor : __ xorr(x10, x11, x10); break;
1295     default   : ShouldNotReachHere();
1296   }
1297 }
1298 
1299 void TemplateTable::idiv() {
1300   transition(itos, itos);
1301   // explicitly check for div0
1302   Label no_div0;
1303   __ bnez(x10, no_div0);
1304   __ mv(t1, Interpreter::_throw_ArithmeticException_entry);
1305   __ jr(t1);
1306   __ bind(no_div0);
1307   __ pop_i(x11);
1308   // x10 <== x11 idiv x10
1309   __ divw(x10, x11, x10);
1310 }
1311 
1312 void TemplateTable::irem() {
1313   transition(itos, itos);
1314   // explicitly check for div0
1315   Label no_div0;
1316   __ bnez(x10, no_div0);
1317   __ mv(t1, Interpreter::_throw_ArithmeticException_entry);
1318   __ jr(t1);
1319   __ bind(no_div0);
1320   __ pop_i(x11);
1321   // x10 <== x11 irem x10
1322   __ remw(x10, x11, x10);
1323 }
1324 
1325 void TemplateTable::lmul() {
1326   transition(ltos, ltos);
1327   __ pop_l(x11);
1328   __ mul(x10, x10, x11);
1329 }
1330 
1331 void TemplateTable::ldiv() {
1332   transition(ltos, ltos);
1333   // explicitly check for div0
1334   Label no_div0;
1335   __ bnez(x10, no_div0);
1336   __ mv(t1, Interpreter::_throw_ArithmeticException_entry);
1337   __ jr(t1);
1338   __ bind(no_div0);
1339   __ pop_l(x11);
1340   // x10 <== x11 ldiv x10
1341   __ div(x10, x11, x10);
1342 }
1343 
1344 void TemplateTable::lrem() {
1345   transition(ltos, ltos);
1346   // explicitly check for div0
1347   Label no_div0;
1348   __ bnez(x10, no_div0);
1349   __ mv(t1, Interpreter::_throw_ArithmeticException_entry);
1350   __ jr(t1);
1351   __ bind(no_div0);
1352   __ pop_l(x11);
1353   // x10 <== x11 lrem x10
1354   __ rem(x10, x11, x10);
1355 }
1356 
1357 void TemplateTable::lshl() {
1358   transition(itos, ltos);
1359   // shift count is in x10
1360   __ pop_l(x11);
1361   __ sll(x10, x11, x10);
1362 }
1363 
1364 void TemplateTable::lshr() {
1365   transition(itos, ltos);
1366   // shift count is in x10
1367   __ pop_l(x11);
1368   __ sra(x10, x11, x10);
1369 }
1370 
1371 void TemplateTable::lushr() {
1372   transition(itos, ltos);
1373   // shift count is in x10
1374   __ pop_l(x11);
1375   __ srl(x10, x11, x10);
1376 }
1377 
1378 void TemplateTable::fop2(Operation op) {
1379   transition(ftos, ftos);
1380   switch (op) {
1381     case add:
1382       __ pop_f(f11);
1383       __ fadd_s(f10, f11, f10);
1384       break;
1385     case sub:
1386       __ pop_f(f11);
1387       __ fsub_s(f10, f11, f10);
1388       break;
1389     case mul:
1390       __ pop_f(f11);
1391       __ fmul_s(f10, f11, f10);
1392       break;
1393     case div:
1394       __ pop_f(f11);
1395       __ fdiv_s(f10, f11, f10);
1396       break;
1397     case rem:
1398       __ fmv_s(f11, f10);
1399       __ pop_f(f10);
1400       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem));
1401       break;
1402     default:
1403       ShouldNotReachHere();
1404   }
1405 }
1406 
1407 void TemplateTable::dop2(Operation op) {
1408   transition(dtos, dtos);
1409   switch (op) {
1410     case add:
1411       __ pop_d(f11);
1412       __ fadd_d(f10, f11, f10);
1413       break;
1414     case sub:
1415       __ pop_d(f11);
1416       __ fsub_d(f10, f11, f10);
1417       break;
1418     case mul:
1419       __ pop_d(f11);
1420       __ fmul_d(f10, f11, f10);
1421       break;
1422     case div:
1423       __ pop_d(f11);
1424       __ fdiv_d(f10, f11, f10);
1425       break;
1426     case rem:
1427       __ fmv_d(f11, f10);
1428       __ pop_d(f10);
1429       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem));
1430       break;
1431     default:
1432       ShouldNotReachHere();
1433   }
1434 }
1435 
1436 void TemplateTable::ineg() {
1437   transition(itos, itos);
1438   __ negw(x10, x10);
1439 }
1440 
1441 void TemplateTable::lneg() {
1442   transition(ltos, ltos);
1443   __ neg(x10, x10);
1444 }
1445 
1446 void TemplateTable::fneg() {
1447   transition(ftos, ftos);
1448   __ fneg_s(f10, f10);
1449 }
1450 
1451 void TemplateTable::dneg() {
1452   transition(dtos, dtos);
1453   __ fneg_d(f10, f10);
1454 }
1455 
1456 void TemplateTable::iinc() {
1457   transition(vtos, vtos);
1458   __ load_signed_byte(x11, at_bcp(2)); // get constant
1459   locals_index(x12);
1460   __ ld(x10, iaddress(x12, x10, _masm));
1461   __ addw(x10, x10, x11);
1462   __ sd(x10, iaddress(x12, t0, _masm));
1463 }
1464 
1465 void TemplateTable::wide_iinc() {
1466   transition(vtos, vtos);
1467   // get constant
1468   // Convert the 16-bit value into native byte-ordering and sign-extend
1469   __ lb(x11, at_bcp(4));
1470   __ lbu(t1, at_bcp(5));
1471   __ slli(x11, x11, 8);
1472   __ orr(x11, x11, t1);
1473 
1474   locals_index_wide(x12);
1475   __ ld(x10, iaddress(x12, t0, _masm));
1476   __ addw(x10, x10, x11);
1477   __ sd(x10, iaddress(x12, t0, _masm));
1478 }
1479 
1480 void TemplateTable::convert() {
1481   // Checking
1482 #ifdef ASSERT
1483   {
1484     TosState tos_in  = ilgl;
1485     TosState tos_out = ilgl;
1486     switch (bytecode()) {
1487       case Bytecodes::_i2l: // fall through
1488       case Bytecodes::_i2f: // fall through
1489       case Bytecodes::_i2d: // fall through
1490       case Bytecodes::_i2b: // fall through
1491       case Bytecodes::_i2c: // fall through
1492       case Bytecodes::_i2s: tos_in = itos; break;
1493       case Bytecodes::_l2i: // fall through
1494       case Bytecodes::_l2f: // fall through
1495       case Bytecodes::_l2d: tos_in = ltos; break;
1496       case Bytecodes::_f2i: // fall through
1497       case Bytecodes::_f2l: // fall through
1498       case Bytecodes::_f2d: tos_in = ftos; break;
1499       case Bytecodes::_d2i: // fall through
1500       case Bytecodes::_d2l: // fall through
1501       case Bytecodes::_d2f: tos_in = dtos; break;
1502       default             : ShouldNotReachHere();
1503     }
1504     switch (bytecode()) {
1505       case Bytecodes::_l2i: // fall through
1506       case Bytecodes::_f2i: // fall through
1507       case Bytecodes::_d2i: // fall through
1508       case Bytecodes::_i2b: // fall through
1509       case Bytecodes::_i2c: // fall through
1510       case Bytecodes::_i2s: tos_out = itos; break;
1511       case Bytecodes::_i2l: // fall through
1512       case Bytecodes::_f2l: // fall through
1513       case Bytecodes::_d2l: tos_out = ltos; break;
1514       case Bytecodes::_i2f: // fall through
1515       case Bytecodes::_l2f: // fall through
1516       case Bytecodes::_d2f: tos_out = ftos; break;
1517       case Bytecodes::_i2d: // fall through
1518       case Bytecodes::_l2d: // fall through
1519       case Bytecodes::_f2d: tos_out = dtos; break;
1520       default             : ShouldNotReachHere();
1521     }
1522     transition(tos_in, tos_out);
1523   }
1524 #endif // ASSERT
1525 
1526   // Conversion
1527   switch (bytecode()) {
1528     case Bytecodes::_i2l:
1529       __ sext(x10, x10, 32);
1530       break;
1531     case Bytecodes::_i2f:
1532       __ fcvt_s_w(f10, x10);
1533       break;
1534     case Bytecodes::_i2d:
1535       __ fcvt_d_w(f10, x10);
1536       break;
1537     case Bytecodes::_i2b:
1538       __ sext(x10, x10, 8);
1539       break;
1540     case Bytecodes::_i2c:
1541       __ zext(x10, x10, 16);
1542       break;
1543     case Bytecodes::_i2s:
1544       __ sext(x10, x10, 16);
1545       break;
1546     case Bytecodes::_l2i:
1547       __ sext(x10, x10, 32);
1548       break;
1549     case Bytecodes::_l2f:
1550       __ fcvt_s_l(f10, x10);
1551       break;
1552     case Bytecodes::_l2d:
1553       __ fcvt_d_l(f10, x10);
1554       break;
1555     case Bytecodes::_f2i:
1556       __ fcvt_w_s_safe(x10, f10);
1557       break;
1558     case Bytecodes::_f2l:
1559       __ fcvt_l_s_safe(x10, f10);
1560       break;
1561     case Bytecodes::_f2d:
1562       __ fcvt_d_s(f10, f10);
1563       break;
1564     case Bytecodes::_d2i:
1565       __ fcvt_w_d_safe(x10, f10);
1566       break;
1567     case Bytecodes::_d2l:
1568       __ fcvt_l_d_safe(x10, f10);
1569       break;
1570     case Bytecodes::_d2f:
1571       __ fcvt_s_d(f10, f10);
1572       break;
1573     default:
1574       ShouldNotReachHere();
1575   }
1576 }
1577 
1578 void TemplateTable::lcmp() {
1579   transition(ltos, itos);
1580   __ pop_l(x11);
1581   __ cmp_l2i(t0, x11, x10);
1582   __ mv(x10, t0);
1583 }
1584 
1585 void TemplateTable::float_cmp(bool is_float, int unordered_result) {
1586   // For instruction feq, flt and fle, the result is 0 if either operand is NaN
1587   if (is_float) {
1588     __ pop_f(f11);
1589     // if unordered_result < 0:
1590     //   we want -1 for unordered or less than, 0 for equal and 1 for
1591     //   greater than.
1592     // else:
1593     //   we want -1 for less than, 0 for equal and 1 for unordered or
1594     //   greater than.
1595     // f11 primary, f10 secondary
1596     __ float_compare(x10, f11, f10, unordered_result);
1597   } else {
1598     __ pop_d(f11);
1599     // if unordered_result < 0:
1600     //   we want -1 for unordered or less than, 0 for equal and 1 for
1601     //   greater than.
1602     // else:
1603     //   we want -1 for less than, 0 for equal and 1 for unordered or
1604     //   greater than.
1605     // f11 primary, f10 secondary
1606     __ double_compare(x10, f11, f10, unordered_result);
1607   }
1608 }
1609 
1610 void TemplateTable::branch(bool is_jsr, bool is_wide) {
1611   __ profile_taken_branch(x10);
1612   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1613                              InvocationCounter::counter_offset();
1614   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1615                               InvocationCounter::counter_offset();
1616 
1617   // load branch displacement
1618   if (!is_wide) {
1619     // Convert the 16-bit value into native byte-ordering and sign-extend
1620     __ lb(x12, at_bcp(1));
1621     __ lbu(t1, at_bcp(2));
1622     __ slli(x12, x12, 8);
1623     __ orr(x12, x12, t1);
1624   } else {
1625     __ lwu(x12, at_bcp(1));
1626     __ revbw(x12, x12);
1627   }
1628 
1629   // Handle all the JSR stuff here, then exit.
1630   // It's much shorter and cleaner than intermingling with the non-JSR
1631   // normal-branch stuff occurring below.
1632 
1633   if (is_jsr) {
1634     // compute return address as bci
1635     __ ld(t1, Address(xmethod, Method::const_offset()));
1636     __ add(t1, t1,
1637            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1638     __ sub(x11, xbcp, t1);
1639     __ push_i(x11);
1640     // Adjust the bcp by the 16-bit displacement in x12
1641     __ add(xbcp, xbcp, x12);
1642     __ load_unsigned_byte(t0, Address(xbcp, 0));
1643     // load the next target bytecode into t0, it is the argument of dispatch_only
1644     __ dispatch_only(vtos, /*generate_poll*/true);
1645     return;
1646   }
1647 
1648   // Normal (non-jsr) branch handling
1649 
1650   // Adjust the bcp by the displacement in x12
1651   __ add(xbcp, xbcp, x12);
1652 
1653   assert(UseLoopCounter || !UseOnStackReplacement,
1654          "on-stack-replacement requires loop counters");
1655   Label backedge_counter_overflow;
1656   Label dispatch;
1657   if (UseLoopCounter) {
1658     // increment backedge counter for backward branches
1659     // x10: MDO
1660     // x12: target offset
1661     __ bgtz(x12, dispatch); // count only if backward branch
1662 
1663     // check if MethodCounters exists
1664     Label has_counters;
1665     __ ld(t0, Address(xmethod, Method::method_counters_offset()));
1666     __ bnez(t0, has_counters);
1667     __ push_reg(x10);
1668     __ push_reg(x12);
1669     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1670             InterpreterRuntime::build_method_counters), xmethod);
1671     __ pop_reg(x12);
1672     __ pop_reg(x10);
1673     __ ld(t0, Address(xmethod, Method::method_counters_offset()));
1674     __ beqz(t0, dispatch); // No MethodCounters allocated, OutOfMemory
1675     __ bind(has_counters);
1676 
1677     Label no_mdo;
1678     int increment = InvocationCounter::count_increment;
1679     if (ProfileInterpreter) {
1680       // Are we profiling?
1681       __ ld(x11, Address(xmethod, in_bytes(Method::method_data_offset())));
1682       __ beqz(x11, no_mdo);
1683       // Increment the MDO backedge counter
1684       const Address mdo_backedge_counter(x11, in_bytes(MethodData::backedge_counter_offset()) +
1685                                          in_bytes(InvocationCounter::counter_offset()));
1686       const Address mask(x11, in_bytes(MethodData::backedge_mask_offset()));
1687       __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1688                                  x10, t0, false,
1689                                  UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1690       __ j(dispatch);
1691     }
1692     __ bind(no_mdo);
1693     // Increment backedge counter in MethodCounters*
1694     __ ld(t0, Address(xmethod, Method::method_counters_offset()));
1695     const Address mask(t0, in_bytes(MethodCounters::backedge_mask_offset()));
1696     __ increment_mask_and_jump(Address(t0, be_offset), increment, mask,
1697                                x10, t1, false,
1698                                UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1699     __ bind(dispatch);
1700   }
1701 
1702   // Pre-load the next target bytecode into t0
1703   __ load_unsigned_byte(t0, Address(xbcp, 0));
1704 
1705   // continue with the bytecode @ target
1706   // t0: target bytecode
1707   // xbcp: target bcp
1708   __ dispatch_only(vtos, /*generate_poll*/true);
1709 
1710   if (UseLoopCounter && UseOnStackReplacement) {
1711     // invocation counter overflow
1712     __ bind(backedge_counter_overflow);
1713     __ neg(x12, x12);
1714     __ add(x12, x12, xbcp);     // branch xbcp
1715     // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
1716     __ call_VM(noreg,
1717                CAST_FROM_FN_PTR(address,
1718                                 InterpreterRuntime::frequency_counter_overflow),
1719                x12);
1720     __ load_unsigned_byte(x11, Address(xbcp, 0));  // restore target bytecode
1721 
1722     // x10: osr nmethod (osr ok) or null (osr not possible)
1723     // w11: target bytecode
1724     // x12: temporary
1725     __ beqz(x10, dispatch);     // test result -- no osr if null
1726     // nmethod may have been invalidated (VM may block upon call_VM return)
1727     __ lbu(x12, Address(x10, nmethod::state_offset()));
1728     if (nmethod::in_use != 0) {
1729       __ sub(x12, x12, nmethod::in_use);
1730     }
1731     __ bnez(x12, dispatch);
1732 
1733     // We have the address of an on stack replacement routine in x10
1734     // We need to prepare to execute the OSR method. First we must
1735     // migrate the locals and monitors off of the stack.
1736 
1737     __ mv(x9, x10);                             // save the nmethod
1738 
1739     JFR_ONLY(__ enter_jfr_critical_section();)
1740 
1741     call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
1742 
1743     // x10 is OSR buffer, move it to expected parameter location
1744     __ mv(j_rarg0, x10);
1745 
1746     // remove activation
1747     // get sender esp
1748     __ ld(esp,
1749           Address(fp, frame::interpreter_frame_sender_sp_offset * wordSize));
1750     // remove frame anchor
1751     __ leave();
1752 
1753     JFR_ONLY(__ leave_jfr_critical_section();)
1754 
1755     // Ensure compiled code always sees stack at proper alignment
1756     __ andi(sp, esp, -16);
1757 
1758     // and begin the OSR nmethod
1759     __ ld(t1, Address(x9, nmethod::osr_entry_point_offset()));
1760     __ jr(t1);
1761   }
1762 }
1763 
1764 void TemplateTable::if_0cmp(Condition cc) {
1765   transition(itos, vtos);
1766   // assume branch is more often taken than not (loops use backward branches)
1767   Label not_taken;
1768 
1769   __ sext(x10, x10, 32);
1770   switch (cc) {
1771     case equal:
1772       __ bnez(x10, not_taken);
1773       break;
1774     case not_equal:
1775       __ beqz(x10, not_taken);
1776       break;
1777     case less:
1778       __ bgez(x10, not_taken);
1779       break;
1780     case less_equal:
1781       __ bgtz(x10, not_taken);
1782       break;
1783     case greater:
1784       __ blez(x10, not_taken);
1785       break;
1786     case greater_equal:
1787       __ bltz(x10, not_taken);
1788       break;
1789     default:
1790       break;
1791   }
1792 
1793   branch(false, false);
1794   __ bind(not_taken);
1795   __ profile_not_taken_branch(x10);
1796 }
1797 
1798 void TemplateTable::if_icmp(Condition cc) {
1799   transition(itos, vtos);
1800   // assume branch is more often taken than not (loops use backward branches)
1801   Label not_taken;
1802   __ pop_i(x11);
1803   __ sext(x10, x10, 32);
1804   switch (cc) {
1805     case equal:
1806       __ bne(x11, x10, not_taken);
1807       break;
1808     case not_equal:
1809       __ beq(x11, x10, not_taken);
1810       break;
1811     case less:
1812       __ bge(x11, x10, not_taken);
1813       break;
1814     case less_equal:
1815       __ bgt(x11, x10, not_taken);
1816       break;
1817     case greater:
1818       __ ble(x11, x10, not_taken);
1819       break;
1820     case greater_equal:
1821       __ blt(x11, x10, not_taken);
1822       break;
1823     default:
1824       break;
1825   }
1826 
1827   branch(false, false);
1828   __ bind(not_taken);
1829   __ profile_not_taken_branch(x10);
1830 }
1831 
1832 void TemplateTable::if_nullcmp(Condition cc) {
1833   transition(atos, vtos);
1834   // assume branch is more often taken than not (loops use backward branches)
1835   Label not_taken;
1836   if (cc == equal) {
1837     __ bnez(x10, not_taken);
1838   } else {
1839     __ beqz(x10, not_taken);
1840   }
1841   branch(false, false);
1842   __ bind(not_taken);
1843   __ profile_not_taken_branch(x10);
1844 }
1845 
1846 void TemplateTable::if_acmp(Condition cc) {
1847   transition(atos, vtos);
1848   // assume branch is more often taken than not (loops use backward branches)
1849   Label not_taken;
1850   __ pop_ptr(x11);
1851 
1852   if (cc == equal) {
1853     __ bne(x11, x10, not_taken);
1854   } else if (cc == not_equal) {
1855     __ beq(x11, x10, not_taken);
1856   }
1857   branch(false, false);
1858   __ bind(not_taken);
1859   __ profile_not_taken_branch(x10);
1860 }
1861 
1862 void TemplateTable::ret() {
1863   transition(vtos, vtos);
1864   locals_index(x11);
1865   __ ld(x11, aaddress(x11, t1, _masm)); // get return bci, compute return bcp
1866   __ profile_ret(x11, x12);
1867   __ ld(xbcp, Address(xmethod, Method::const_offset()));
1868   __ add(xbcp, xbcp, x11);
1869   __ add(xbcp, xbcp, in_bytes(ConstMethod::codes_offset()));
1870   __ dispatch_next(vtos, 0, /*generate_poll*/true);
1871 }
1872 
1873 void TemplateTable::wide_ret() {
1874   transition(vtos, vtos);
1875   locals_index_wide(x11);
1876   __ ld(x11, aaddress(x11, t0, _masm)); // get return bci, compute return bcp
1877   __ profile_ret(x11, x12);
1878   __ ld(xbcp, Address(xmethod, Method::const_offset()));
1879   __ add(xbcp, xbcp, x11);
1880   __ add(xbcp, xbcp, in_bytes(ConstMethod::codes_offset()));
1881   __ dispatch_next(vtos, 0, /*generate_poll*/true);
1882 }
1883 
1884 void TemplateTable::tableswitch() {
1885   Label default_case, continue_execution;
1886   transition(itos, vtos);
1887   // align xbcp
1888   __ la(x11, at_bcp(BytesPerInt));
1889   __ andi(x11, x11, -BytesPerInt);
1890   // load lo & hi
1891   __ lwu(x12, Address(x11, BytesPerInt));
1892   __ lwu(x13, Address(x11, 2 * BytesPerInt));
1893   __ revbw(x12, x12);
1894   __ revbw(x13, x13);
1895   // check against lo & hi
1896   __ blt(x10, x12, default_case);
1897   __ bgt(x10, x13, default_case);
1898   // lookup dispatch offset
1899   __ subw(x10, x10, x12);
1900   __ shadd(x13, x10, x11, t0, 2);
1901   __ lwu(x13, Address(x13, 3 * BytesPerInt));
1902   __ profile_switch_case(x10, x11, x12);
1903   // continue execution
1904   __ bind(continue_execution);
1905   __ revbw(x13, x13);
1906   __ add(xbcp, xbcp, x13);
1907   __ load_unsigned_byte(t0, Address(xbcp));
1908   __ dispatch_only(vtos, /*generate_poll*/true);
1909   // handle default
1910   __ bind(default_case);
1911   __ profile_switch_default(x10);
1912   __ lwu(x13, Address(x11, 0));
1913   __ j(continue_execution);
1914 }
1915 
1916 void TemplateTable::lookupswitch() {
1917   transition(itos, itos);
1918   __ stop("lookupswitch bytecode should have been rewritten");
1919 }
1920 
1921 void TemplateTable::fast_linearswitch() {
1922   transition(itos, vtos);
1923   Label loop_entry, loop, found, continue_execution;
1924   // bswap x10 so we can avoid bswapping the table entries
1925   __ revbw(x10, x10);
1926   // align xbcp
1927   __ la(x9, at_bcp(BytesPerInt)); // btw: should be able to get rid of
1928                                     // this instruction (change offsets
1929                                     // below)
1930   __ andi(x9, x9, -BytesPerInt);
1931   // set counter
1932   __ lwu(x11, Address(x9, BytesPerInt));
1933   // Convert the 32-bit npairs (number of pairs) into native byte-ordering
1934   // We can use sign-extension here because npairs must be greater than or
1935   // equal to 0 per JVM spec on 'lookupswitch' bytecode.
1936   __ revbw(x11, x11);
1937   __ j(loop_entry);
1938   // table search
1939   __ bind(loop);
1940   __ shadd(t0, x11, x9, t0, 3);
1941   __ lw(t0, Address(t0, 2 * BytesPerInt));
1942   __ beq(x10, t0, found);
1943   __ bind(loop_entry);
1944   __ subi(x11, x11, 1);
1945   __ bgez(x11, loop);
1946   // default case
1947   __ profile_switch_default(x10);
1948   __ lwu(x13, Address(x9, 0));
1949   __ j(continue_execution);
1950   // entry found -> get offset
1951   __ bind(found);
1952   __ shadd(t0, x11, x9, t0, 3);
1953   __ lwu(x13, Address(t0, 3 * BytesPerInt));
1954   __ profile_switch_case(x11, x10, x9);
1955   // continue execution
1956   __ bind(continue_execution);
1957   __ revbw(x13, x13);
1958   __ add(xbcp, xbcp, x13);
1959   __ lbu(t0, Address(xbcp, 0));
1960   __ dispatch_only(vtos, /*generate_poll*/true);
1961 }
1962 
1963 void TemplateTable::fast_binaryswitch() {
1964   transition(itos, vtos);
1965   // Implementation using the following core algorithm:
1966   //
1967   // int binary_search(int key, LookupswitchPair* array, int n)
1968   //   binary_search start:
1969   //   #Binary search according to "Methodik des Programmierens" by
1970   //   # Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
1971   //   int i = 0;
1972   //   int j = n;
1973   //   while (i + 1 < j) do
1974   //     # invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
1975   //     # with      Q: for all i: 0 <= i < n: key < a[i]
1976   //     # where a stands for the array and assuming that the (inexisting)
1977   //     # element a[n] is infinitely big.
1978   //     int h = (i + j) >> 1
1979   //     # i < h < j
1980   //     if (key < array[h].fast_match())
1981   //     then [j = h]
1982   //     else [i = h]
1983   //   end
1984   //   # R: a[i] <= key < a[i+1] or Q
1985   //   # (i.e., if key is within array, i is the correct index)
1986   //   return i
1987   // binary_search end
1988 
1989 
1990   // Register allocation
1991   const Register key   = x10; // already set (tosca)
1992   const Register array = x11;
1993   const Register i     = x12;
1994   const Register j     = x13;
1995   const Register h     = x14;
1996   const Register temp  = x15;
1997 
1998   // Find array start
1999   __ la(array, at_bcp(3 * BytesPerInt));  // btw: should be able to
2000                                           // get rid of this
2001                                           // instruction (change
2002                                           // offsets below)
2003   __ andi(array, array, -BytesPerInt);
2004 
2005   // Initialize i & j
2006   __ mv(i, zr);                            // i = 0
2007   __ lwu(j, Address(array, -BytesPerInt)); // j = length(array)
2008 
2009   // Convert the 32-bit npairs (number of pairs) into native byte-ordering
2010   // We can use sign-extension here because npairs must be greater than or
2011   // equal to 0 per JVM spec on 'lookupswitch' bytecode.
2012   __ revbw(j, j);
2013 
2014   // And start
2015   Label entry;
2016   __ j(entry);
2017 
2018   // binary search loop
2019   {
2020     Label loop;
2021     __ bind(loop);
2022     __ addw(h, i, j);                           // h = i + j
2023     __ srliw(h, h, 1);                          // h = (i + j) >> 1
2024     // if [key < array[h].fast_match()]
2025     // then [j = h]
2026     // else [i = h]
2027     // Convert array[h].match to native byte-ordering before compare
2028     __ shadd(temp, h, array, temp, 3);
2029     __ lwu(temp, Address(temp, 0));
2030     __ revbw(temp, temp);
2031 
2032     Label L_done, L_greater;
2033     __ bge(key, temp, L_greater);
2034     // if [key < array[h].fast_match()] then j = h
2035     __ mv(j, h);
2036     __ j(L_done);
2037     __ bind(L_greater);
2038     // if [key >= array[h].fast_match()] then i = h
2039     __ mv(i, h);
2040     __ bind(L_done);
2041 
2042     // while [i + 1 < j]
2043     __ bind(entry);
2044     __ addiw(h, i, 1);         // i + 1
2045     __ blt(h, j, loop);        // i + 1 < j
2046   }
2047 
2048   // end of binary search, result index is i (must check again!)
2049   Label default_case;
2050   // Convert array[i].match to native byte-ordering before compare
2051   __ shadd(temp, i, array, temp, 3);
2052   __ lwu(temp, Address(temp, 0));
2053   __ revbw(temp, temp);
2054   __ bne(key, temp, default_case);
2055 
2056   // entry found -> j = offset
2057   __ shadd(temp, i, array, temp, 3);
2058   __ lwu(j, Address(temp, BytesPerInt));
2059   __ profile_switch_case(i, key, array);
2060   __ revbw(j, j);
2061 
2062   __ add(temp, xbcp, j);
2063   __ load_unsigned_byte(t0, Address(temp, 0));
2064 
2065   __ add(xbcp, xbcp, j);
2066   __ la(xbcp, Address(xbcp, 0));
2067   __ dispatch_only(vtos, /*generate_poll*/true);
2068 
2069   // default case -> j = default offset
2070   __ bind(default_case);
2071   __ profile_switch_default(i);
2072   __ lwu(j, Address(array, -2 * BytesPerInt));
2073   __ revbw(j, j);
2074 
2075   __ add(temp, xbcp, j);
2076   __ load_unsigned_byte(t0, Address(temp, 0));
2077 
2078   __ add(xbcp, xbcp, j);
2079   __ la(xbcp, Address(xbcp, 0));
2080   __ dispatch_only(vtos, /*generate_poll*/true);
2081 }
2082 
2083 void TemplateTable::_return(TosState state) {
2084   transition(state, state);
2085   assert(_desc->calls_vm(),
2086          "inconsistent calls_vm information"); // call in remove_activation
2087 
2088   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2089     assert(state == vtos, "only valid state");
2090 
2091     __ ld(c_rarg1, aaddress(0));
2092     __ load_klass(x13, c_rarg1);
2093     __ lbu(x13, Address(x13, Klass::misc_flags_offset()));
2094     Label skip_register_finalizer;
2095     __ test_bit(t0, x13, exact_log2(KlassFlags::_misc_has_finalizer));
2096     __ beqz(t0, skip_register_finalizer);
2097 
2098     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2099 
2100     __ bind(skip_register_finalizer);
2101   }
2102 
2103   // Issue a StoreStore barrier after all stores but before return
2104   // from any constructor for any class with a final field. We don't
2105   // know if this is a finalizer, so we always do so.
2106   if (_desc->bytecode() == Bytecodes::_return) {
2107     __ membar(MacroAssembler::StoreStore);
2108   }
2109 
2110   if (_desc->bytecode() != Bytecodes::_return_register_finalizer) {
2111     Label no_safepoint;
2112     __ ld(t0, Address(xthread, JavaThread::polling_word_offset()));
2113     __ test_bit(t0, t0, exact_log2(SafepointMechanism::poll_bit()));
2114     __ beqz(t0, no_safepoint);
2115     __ push(state);
2116     __ push_cont_fastpath(xthread);
2117     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::at_safepoint));
2118     __ pop_cont_fastpath(xthread);
2119     __ pop(state);
2120     __ bind(no_safepoint);
2121   }
2122 
2123   // Narrow result if state is itos but result type is smaller.
2124   // Need to narrow in the return bytecode rather than in generate_return_entry
2125   // since compiled code callers expect the result to already be narrowed.
2126   if (state == itos) {
2127     __ narrow(x10);
2128   }
2129 
2130   __ remove_activation(state);
2131   __ ret();
2132 }
2133 
2134 
2135 // ----------------------------------------------------------------------------
2136 // Volatile variables demand their effects be made known to all CPU's
2137 // in order.  Store buffers on most chips allow reads & writes to
2138 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2139 // without some kind of memory barrier (i.e., it's not sufficient that
2140 // the interpreter does not reorder volatile references, the hardware
2141 // also must not reorder them).
2142 //
2143 // According to the new Java Memory Model (JMM):
2144 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2145 //     writes act as acquire & release, so:
2146 // (2) A read cannot let unrelated NON-volatile memory refs that
2147 //     happen after the read float up to before the read.  It's OK for
2148 //     non-volatile memory refs that happen before the volatile read to
2149 //     float down below it.
2150 // (3) Similar a volatile write cannot let unrelated NON-volatile
2151 //     memory refs that happen BEFORE the write float down to after the
2152 //     write.  It's OK for non-volatile memory refs that happen after the
2153 //     volatile write to float up before it.
2154 //
2155 // We only put in barriers around volatile refs (they are expensive),
2156 // not _between_ memory refs (that would require us to track the
2157 // flavor of the previous memory refs).  Requirements (2) and (3)
2158 // require some barriers before volatile stores and after volatile
2159 // loads.  These nearly cover requirement (1) but miss the
2160 // volatile-store-volatile-load case.  This final case is placed after
2161 // volatile-stores although it could just as well go before
2162 // volatile-loads.
2163 
2164 void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
2165                                                        Register Rcache,
2166                                                        Register index) {
2167   const Register temp = x9; // s1
2168   assert_different_registers(Rcache, index, temp);
2169   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2170 
2171   Label resolved, clinit_barrier_slow;
2172 
2173   Bytecodes::Code code = bytecode();
2174   __ load_method_entry(Rcache, index);
2175   switch(byte_no) {
2176     case f1_byte:
2177       __ add(temp, Rcache, in_bytes(ResolvedMethodEntry::bytecode1_offset()));
2178       break;
2179     case f2_byte:
2180       __ add(temp, Rcache, in_bytes(ResolvedMethodEntry::bytecode2_offset()));
2181       break;
2182   }
2183   // Load-acquire the bytecode to match store-release in InterpreterRuntime
2184   __ lbu(temp, Address(temp, 0));
2185   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2186 
2187   __ mv(t0, (int) code);
2188   __ beq(temp, t0, resolved);  // have we resolved this bytecode?
2189 
2190   // resolve first time through
2191   // Class initialization barrier slow path lands here as well.
2192   __ bind(clinit_barrier_slow);
2193 
2194   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2195   __ mv(temp, (int) code);
2196   __ call_VM(noreg, entry, temp);
2197 
2198   // Update registers with resolved info
2199   __ load_method_entry(Rcache, index);
2200   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2201   // so all clients ofthis method must be modified accordingly
2202   __ bind(resolved);
2203 
2204   // Class initialization barrier for static methods
2205   if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
2206     __ ld(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
2207     __ load_method_holder(temp, temp);
2208     __ clinit_barrier(temp, t0, nullptr, &clinit_barrier_slow);
2209   }
2210 }
2211 
2212 void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
2213                                             Register Rcache,
2214                                             Register index) {
2215   const Register temp = x9;
2216   assert_different_registers(Rcache, index, temp);
2217 
2218   Label resolved;
2219 
2220   Bytecodes::Code code = bytecode();
2221   switch (code) {
2222   case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2223   case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2224   default: break;
2225   }
2226 
2227   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2228   __ load_field_entry(Rcache, index);
2229   if (byte_no == f1_byte) {
2230     __ la(temp, Address(Rcache, in_bytes(ResolvedFieldEntry::get_code_offset())));
2231   } else {
2232     __ la(temp, Address(Rcache, in_bytes(ResolvedFieldEntry::put_code_offset())));
2233   }
2234   // Load-acquire the bytecode to match store-release in ResolvedFieldEntry::fill_in()
2235   __ lbu(temp, Address(temp, 0));
2236   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2237   __ mv(t0, (int) code);  // have we resolved this bytecode?
2238   __ beq(temp, t0, resolved);
2239 
2240   // resolve first time through
2241   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2242   __ mv(temp, (int) code);
2243   __ call_VM(noreg, entry, temp);
2244 
2245   // Update registers with resolved info
2246   __ load_field_entry(Rcache, index);
2247   __ bind(resolved);
2248 }
2249 
2250 void TemplateTable::load_resolved_field_entry(Register obj,
2251                                               Register cache,
2252                                               Register tos_state,
2253                                               Register offset,
2254                                               Register flags,
2255                                               bool is_static = false) {
2256   assert_different_registers(cache, tos_state, flags, offset);
2257 
2258   // Field offset
2259   __ load_sized_value(offset, Address(cache, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
2260 
2261   // Flags
2262   __ load_unsigned_byte(flags, Address(cache, in_bytes(ResolvedFieldEntry::flags_offset())));
2263 
2264   // TOS state
2265   if (tos_state != noreg) {
2266     __ load_unsigned_byte(tos_state, Address(cache, in_bytes(ResolvedFieldEntry::type_offset())));
2267   }
2268 
2269   // Klass overwrite register
2270   if (is_static) {
2271     __ ld(obj, Address(cache, ResolvedFieldEntry::field_holder_offset()));
2272     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2273     __ ld(obj, Address(obj, mirror_offset));
2274     __ resolve_oop_handle(obj, x15, t1);
2275   }
2276 }
2277 
2278 void TemplateTable::load_resolved_method_entry_special_or_static(Register cache,
2279                                                                  Register method,
2280                                                                  Register flags) {
2281 
2282   // setup registers
2283   const Register index = flags;
2284   assert_different_registers(method, cache, flags);
2285 
2286   // determine constant pool cache field offsets
2287   resolve_cache_and_index_for_method(f1_byte, cache, index);
2288   __ load_unsigned_byte(flags, Address(cache, in_bytes(ResolvedMethodEntry::flags_offset())));
2289   __ ld(method, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
2290 }
2291 
2292 void TemplateTable::load_resolved_method_entry_handle(Register cache,
2293                                                       Register method,
2294                                                       Register ref_index,
2295                                                       Register flags) {
2296   // setup registers
2297   const Register index = ref_index;
2298   assert_different_registers(method, flags);
2299   assert_different_registers(method, cache, index);
2300 
2301   // determine constant pool cache field offsets
2302   resolve_cache_and_index_for_method(f1_byte, cache, index);
2303   __ load_unsigned_byte(flags, Address(cache, in_bytes(ResolvedMethodEntry::flags_offset())));
2304 
2305   // maybe push appendix to arguments (just before return address)
2306   Label L_no_push;
2307   __ test_bit(t0, flags, ResolvedMethodEntry::has_appendix_shift);
2308   __ beqz(t0, L_no_push);
2309   // invokehandle uses an index into the resolved references array
2310   __ load_unsigned_short(ref_index, Address(cache, in_bytes(ResolvedMethodEntry::resolved_references_index_offset())));
2311   // Push the appendix as a trailing parameter.
2312   // This must be done before we get the receiver,
2313   // since the parameter_size includes it.
2314   Register appendix = method;
2315   __ load_resolved_reference_at_index(appendix, ref_index);
2316   __ push_reg(appendix); // push appendix (MethodType, CallSite, etc.)
2317   __ bind(L_no_push);
2318 
2319   __ ld(method, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
2320 }
2321 
2322 void TemplateTable::load_resolved_method_entry_interface(Register cache,
2323                                                          Register klass,
2324                                                          Register method_or_table_index,
2325                                                          Register flags) {
2326   // setup registers
2327   const Register index = method_or_table_index;
2328   assert_different_registers(method_or_table_index, cache, flags);
2329 
2330   // determine constant pool cache field offsets
2331   resolve_cache_and_index_for_method(f1_byte, cache, index);
2332   __ load_unsigned_byte(flags, Address(cache, in_bytes(ResolvedMethodEntry::flags_offset())));
2333 
2334   // Invokeinterface can behave in different ways:
2335   // If calling a method from java.lang.Object, the forced virtual flag is true so the invocation will
2336   // behave like an invokevirtual call. The state of the virtual final flag will determine whether a method or
2337   // vtable index is placed in the register.
2338   // Otherwise, the registers will be populated with the klass and method.
2339 
2340   Label NotVirtual; Label NotVFinal; Label Done;
2341   __ test_bit(t0, flags, ResolvedMethodEntry::is_forced_virtual_shift);
2342   __ beqz(t0, NotVirtual);
2343   __ test_bit(t0, flags, ResolvedMethodEntry::is_vfinal_shift);
2344   __ beqz(t0, NotVFinal);
2345   __ ld(method_or_table_index, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
2346   __ j(Done);
2347 
2348   __ bind(NotVFinal);
2349   __ load_unsigned_short(method_or_table_index, Address(cache, in_bytes(ResolvedMethodEntry::table_index_offset())));
2350   __ j(Done);
2351 
2352   __ bind(NotVirtual);
2353   __ ld(method_or_table_index, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
2354   __ ld(klass, Address(cache, in_bytes(ResolvedMethodEntry::klass_offset())));
2355   __ bind(Done);
2356 }
2357 
2358 void TemplateTable::load_resolved_method_entry_virtual(Register cache,
2359                                                        Register method_or_table_index,
2360                                                        Register flags) {
2361   // setup registers
2362   const Register index = flags;
2363   assert_different_registers(method_or_table_index, cache, flags);
2364 
2365   // determine constant pool cache field offsets
2366   resolve_cache_and_index_for_method(f2_byte, cache, index);
2367   __ load_unsigned_byte(flags, Address(cache, in_bytes(ResolvedMethodEntry::flags_offset())));
2368 
2369   // method_or_table_index can either be an itable index or a method depending on the virtual final flag
2370   Label NotVFinal; Label Done;
2371   __ test_bit(t0, flags, ResolvedMethodEntry::is_vfinal_shift);
2372   __ beqz(t0, NotVFinal);
2373   __ ld(method_or_table_index, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
2374   __ j(Done);
2375 
2376   __ bind(NotVFinal);
2377   __ load_unsigned_short(method_or_table_index, Address(cache, in_bytes(ResolvedMethodEntry::table_index_offset())));
2378   __ bind(Done);
2379 }
2380 
2381 // The xmethod register is input and overwritten to be the adapter method for the
2382 // indy call. Return address (ra) is set to the return address for the adapter and
2383 // an appendix may be pushed to the stack. Registers x10-x13 are clobbered.
2384 void TemplateTable::load_invokedynamic_entry(Register method) {
2385   // setup registers
2386   const Register appendix = x10;
2387   const Register cache = x12;
2388   const Register index = x13;
2389   assert_different_registers(method, appendix, cache, index, xcpool);
2390 
2391   __ save_bcp();
2392 
2393   Label resolved;
2394 
2395   __ load_resolved_indy_entry(cache, index);
2396   __ ld(method, Address(cache, in_bytes(ResolvedIndyEntry::method_offset())));
2397   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2398 
2399   // Compare the method to zero
2400   __ bnez(method, resolved);
2401 
2402   Bytecodes::Code code = bytecode();
2403 
2404   // Call to the interpreter runtime to resolve invokedynamic
2405   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2406   __ mv(method, code); // this is essentially Bytecodes::_invokedynamic
2407   __ call_VM(noreg, entry, method);
2408   // Update registers with resolved info
2409   __ load_resolved_indy_entry(cache, index);
2410   __ ld(method, Address(cache, in_bytes(ResolvedIndyEntry::method_offset())));
2411   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2412 
2413 #ifdef ASSERT
2414   __ bnez(method, resolved);
2415   __ stop("Should be resolved by now");
2416 #endif // ASSERT
2417   __ bind(resolved);
2418 
2419   Label L_no_push;
2420   // Check if there is an appendix
2421   __ load_unsigned_byte(index, Address(cache, in_bytes(ResolvedIndyEntry::flags_offset())));
2422   __ test_bit(t0, index, ResolvedIndyEntry::has_appendix_shift);
2423   __ beqz(t0, L_no_push);
2424 
2425   // Get appendix
2426   __ load_unsigned_short(index, Address(cache, in_bytes(ResolvedIndyEntry::resolved_references_index_offset())));
2427   // Push the appendix as a trailing parameter
2428   // since the parameter_size includes it.
2429   __ push_reg(method);
2430   __ mv(method, index);
2431   __ load_resolved_reference_at_index(appendix, method);
2432   __ verify_oop(appendix);
2433   __ pop_reg(method);
2434   __ push_reg(appendix);  // push appendix (MethodType, CallSite, etc.)
2435   __ bind(L_no_push);
2436 
2437   // compute return type
2438   __ load_unsigned_byte(index, Address(cache, in_bytes(ResolvedIndyEntry::result_type_offset())));
2439   // load return address
2440   // Return address is loaded into ra and not pushed to the stack like x86
2441   {
2442     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
2443     __ mv(t0, table_addr);
2444     __ shadd(t0, index, t0, index, 3);
2445     __ ld(ra, Address(t0, 0));
2446   }
2447 }
2448 
2449 // The registers cache and index expected to be set before call.
2450 // Correct values of the cache and index registers are preserved.
2451 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2452                                             bool is_static, bool has_tos) {
2453   // do the JVMTI work here to avoid disturbing the register state below
2454   // We use c_rarg registers here because we want to use the register used in
2455   // the call to the VM
2456   if (JvmtiExport::can_post_field_access()) {
2457     // Check to see if a field access watch has been set before we
2458     // take the time to call into the VM.
2459     Label L1;
2460     assert_different_registers(cache, index, x10);
2461     __ lwu(x10, ExternalAddress(JvmtiExport::get_field_access_count_addr()));
2462     __ beqz(x10, L1);
2463 
2464     __ load_field_entry(c_rarg2, index);
2465 
2466     if (is_static) {
2467       __ mv(c_rarg1, zr); // null object reference
2468     } else {
2469       __ ld(c_rarg1, at_tos()); // get object pointer without popping it
2470       __ verify_oop(c_rarg1);
2471     }
2472     // c_rarg1: object pointer or null
2473     // c_rarg2: cache entry pointer
2474     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2475                                        InterpreterRuntime::post_field_access),
2476                                        c_rarg1, c_rarg2);
2477     __ load_field_entry(cache, index);
2478     __ bind(L1);
2479   }
2480 }
2481 
2482 void TemplateTable::pop_and_check_object(Register r) {
2483   __ pop_ptr(r);
2484   __ null_check(r);  // for field access must check obj.
2485   __ verify_oop(r);
2486 }
2487 
2488 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2489   const Register cache     = x14;
2490   const Register obj       = x14;
2491   const Register index     = x13;
2492   const Register tos_state = x13;
2493   const Register off       = x9;
2494   const Register flags     = x16;
2495   const Register bc        = x14; // uses same reg as obj, so don't mix them
2496 
2497   resolve_cache_and_index_for_field(byte_no, cache, index);
2498   jvmti_post_field_access(cache, index, is_static, false);
2499   load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
2500 
2501   if (!is_static) {
2502     // obj is on the stack
2503     pop_and_check_object(obj);
2504   }
2505 
2506   __ add(off, obj, off);
2507   const Address field(off);
2508 
2509   Label Done, notByte, notBool, notInt, notShort, notChar,
2510               notLong, notFloat, notObj, notDouble;
2511 
2512   assert(btos == 0, "change code, btos != 0");
2513   __ bnez(tos_state, notByte);
2514 
2515   // Don't rewrite getstatic, only getfield
2516   if (is_static) {
2517     rc = may_not_rewrite;
2518   }
2519 
2520   // btos
2521   __ access_load_at(T_BYTE, IN_HEAP, x10, field, noreg, noreg);
2522   __ push(btos);
2523   // Rewrite bytecode to be faster
2524   if (rc == may_rewrite) {
2525     patch_bytecode(Bytecodes::_fast_bgetfield, bc, x11);
2526   }
2527   __ j(Done);
2528 
2529   __ bind(notByte);
2530   __ subi(t0, tos_state, (u1)ztos);
2531   __ bnez(t0, notBool);
2532 
2533   // ztos (same code as btos)
2534   __ access_load_at(T_BOOLEAN, IN_HEAP, x10, field, noreg, noreg);
2535   __ push(ztos);
2536   // Rewrite bytecode to be faster
2537   if (rc == may_rewrite) {
2538     // uses btos rewriting, no truncating to t/f bit is needed for getfield
2539     patch_bytecode(Bytecodes::_fast_bgetfield, bc, x11);
2540   }
2541   __ j(Done);
2542 
2543   __ bind(notBool);
2544   __ subi(t0, tos_state, (u1)atos);
2545   __ bnez(t0, notObj);
2546   // atos
2547   __ load_heap_oop(x10, field, x28, x29, IN_HEAP);
2548   __ push(atos);
2549   if (rc == may_rewrite) {
2550     patch_bytecode(Bytecodes::_fast_agetfield, bc, x11);
2551   }
2552   __ j(Done);
2553 
2554   __ bind(notObj);
2555   __ subi(t0, tos_state, (u1)itos);
2556   __ bnez(t0, notInt);
2557   // itos
2558   __ access_load_at(T_INT, IN_HEAP, x10, field, noreg, noreg);
2559   __ sext(x10, x10, 32);
2560   __ push(itos);
2561   // Rewrite bytecode to be faster
2562   if (rc == may_rewrite) {
2563     patch_bytecode(Bytecodes::_fast_igetfield, bc, x11);
2564   }
2565   __ j(Done);
2566 
2567   __ bind(notInt);
2568   __ subi(t0, tos_state, (u1)ctos);
2569   __ bnez(t0, notChar);
2570   // ctos
2571   __ access_load_at(T_CHAR, IN_HEAP, x10, field, noreg, noreg);
2572   __ push(ctos);
2573   // Rewrite bytecode to be faster
2574   if (rc == may_rewrite) {
2575     patch_bytecode(Bytecodes::_fast_cgetfield, bc, x11);
2576   }
2577   __ j(Done);
2578 
2579   __ bind(notChar);
2580   __ subi(t0, tos_state, (u1)stos);
2581   __ bnez(t0, notShort);
2582   // stos
2583   __ access_load_at(T_SHORT, IN_HEAP, x10, field, noreg, noreg);
2584   __ push(stos);
2585   // Rewrite bytecode to be faster
2586   if (rc == may_rewrite) {
2587     patch_bytecode(Bytecodes::_fast_sgetfield, bc, x11);
2588   }
2589   __ j(Done);
2590 
2591   __ bind(notShort);
2592   __ subi(t0, tos_state, (u1)ltos);
2593   __ bnez(t0, notLong);
2594   // ltos
2595   __ access_load_at(T_LONG, IN_HEAP, x10, field, noreg, noreg);
2596   __ push(ltos);
2597   // Rewrite bytecode to be faster
2598   if (rc == may_rewrite) {
2599     patch_bytecode(Bytecodes::_fast_lgetfield, bc, x11);
2600   }
2601   __ j(Done);
2602 
2603   __ bind(notLong);
2604   __ subi(t0, tos_state, (u1)ftos);
2605   __ bnez(t0, notFloat);
2606   // ftos
2607   __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2608   __ push(ftos);
2609   // Rewrite bytecode to be faster
2610   if (rc == may_rewrite) {
2611     patch_bytecode(Bytecodes::_fast_fgetfield, bc, x11);
2612   }
2613   __ j(Done);
2614 
2615   __ bind(notFloat);
2616 #ifdef ASSERT
2617   __ subi(t0, tos_state, (u1)dtos);
2618   __ bnez(t0, notDouble);
2619 #endif
2620   // dtos
2621   __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2622   __ push(dtos);
2623   // Rewrite bytecode to be faster
2624   if (rc == may_rewrite) {
2625     patch_bytecode(Bytecodes::_fast_dgetfield, bc, x11);
2626   }
2627 #ifdef ASSERT
2628   __ j(Done);
2629 
2630   __ bind(notDouble);
2631   __ stop("Bad state");
2632 #endif
2633 
2634   __ bind(Done);
2635 
2636   Label notVolatile;
2637   __ test_bit(t0, flags, ResolvedFieldEntry::is_volatile_shift);
2638   __ beqz(t0, notVolatile);
2639   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2640   __ bind(notVolatile);
2641 }
2642 
2643 void TemplateTable::getfield(int byte_no) {
2644   getfield_or_static(byte_no, false);
2645 }
2646 
2647 void TemplateTable::nofast_getfield(int byte_no) {
2648   getfield_or_static(byte_no, false, may_not_rewrite);
2649 }
2650 
2651 void TemplateTable::getstatic(int byte_no)
2652 {
2653   getfield_or_static(byte_no, true);
2654 }
2655 
2656 // The registers cache and index expected to be set before call.
2657 // The function may destroy various registers, just not the cache and index registers.
2658 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2659   transition(vtos, vtos);
2660 
2661   if (JvmtiExport::can_post_field_modification()) {
2662     // Check to see if a field modification watch has been set before
2663     // we take the time to call into the VM.
2664     Label L1;
2665     assert_different_registers(cache, index, x10);
2666     __ lwu(x10, ExternalAddress(JvmtiExport::get_field_modification_count_addr()));
2667     __ beqz(x10, L1);
2668 
2669     __ mv(c_rarg2, cache);
2670 
2671     if (is_static) {
2672       // Life is simple. Null out the object pointer.
2673       __ mv(c_rarg1, zr);
2674     } else {
2675       // Life is harder. The stack holds the value on top, followed by
2676       // the object. We don't know the size of the value, though; it
2677       // could be one or two words depending on its type. As a result,
2678       // we must find the type to determine where the object is.
2679       __ load_unsigned_byte(c_rarg3, Address(c_rarg2, in_bytes(ResolvedFieldEntry::type_offset())));
2680       Label nope2, done, ok;
2681       __ ld(c_rarg1, at_tos_p1());   // initially assume a one word jvalue
2682       __ subi(t0, c_rarg3, (u1)ltos);
2683       __ beqz(t0, ok);
2684       __ subi(t0, c_rarg3, (u1)dtos);
2685       __ bnez(t0, nope2);
2686       __ bind(ok);
2687       __ ld(c_rarg1, at_tos_p2());  // ltos (two word jvalue);
2688       __ bind(nope2);
2689     }
2690     // object (tos)
2691     __ mv(c_rarg3, esp);
2692     // c_rarg1: object pointer set up above (null if static)
2693     // c_rarg2: cache entry pointer
2694     // c_rarg3: jvalue object on  the stack
2695     __ call_VM(noreg,
2696                CAST_FROM_FN_PTR(address,
2697                                 InterpreterRuntime::post_field_modification),
2698                                 c_rarg1, c_rarg2, c_rarg3);
2699     __ load_field_entry(cache, index);
2700     __ bind(L1);
2701   }
2702 }
2703 
2704 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2705   transition(vtos, vtos);
2706 
2707   const Register cache     = x12;
2708   const Register index     = x13;
2709   const Register tos_state = x13;
2710   const Register obj       = x12;
2711   const Register off       = x9;
2712   const Register flags     = x10;
2713   const Register bc        = x14;
2714 
2715   resolve_cache_and_index_for_field(byte_no, cache, index);
2716   jvmti_post_field_mod(cache, index, is_static);
2717   load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
2718 
2719   Label Done;
2720   __ mv(x15, flags);
2721 
2722   {
2723     Label notVolatile;
2724     __ test_bit(t0, x15, ResolvedFieldEntry::is_volatile_shift);
2725     __ beqz(t0, notVolatile);
2726     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2727     __ bind(notVolatile);
2728   }
2729 
2730   Label notByte, notBool, notInt, notShort, notChar,
2731         notLong, notFloat, notObj, notDouble;
2732 
2733   assert(btos == 0, "change code, btos != 0");
2734   __ bnez(tos_state, notByte);
2735 
2736   // Don't rewrite putstatic, only putfield
2737   if (is_static) {
2738     rc = may_not_rewrite;
2739   }
2740 
2741   // btos
2742   {
2743     __ pop(btos);
2744     // field address
2745     if (!is_static) {
2746       pop_and_check_object(obj);
2747     }
2748     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2749     const Address field(off, 0); // off register as temparator register.
2750     __ access_store_at(T_BYTE, IN_HEAP, field, x10, noreg, noreg, noreg);
2751     if (rc == may_rewrite) {
2752       patch_bytecode(Bytecodes::_fast_bputfield, bc, x11, true, byte_no);
2753     }
2754     __ j(Done);
2755   }
2756 
2757   __ bind(notByte);
2758   __ subi(t0, tos_state, (u1)ztos);
2759   __ bnez(t0, notBool);
2760 
2761   // ztos
2762   {
2763     __ pop(ztos);
2764     // field address
2765     if (!is_static) {
2766       pop_and_check_object(obj);
2767     }
2768     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2769     const Address field(off, 0);
2770     __ access_store_at(T_BOOLEAN, IN_HEAP, field, x10, noreg, noreg, noreg);
2771     if (rc == may_rewrite) {
2772       patch_bytecode(Bytecodes::_fast_zputfield, bc, x11, true, byte_no);
2773     }
2774     __ j(Done);
2775   }
2776 
2777   __ bind(notBool);
2778   __ subi(t0, tos_state, (u1)atos);
2779   __ bnez(t0, notObj);
2780 
2781   // atos
2782   {
2783     __ pop(atos);
2784     // field address
2785     if (!is_static) {
2786       pop_and_check_object(obj);
2787     }
2788     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2789     const Address field(off, 0);
2790     // Store into the field
2791     __ store_heap_oop(field, x10, x28, x29, x13, IN_HEAP);
2792     if (rc == may_rewrite) {
2793       patch_bytecode(Bytecodes::_fast_aputfield, bc, x11, true, byte_no);
2794     }
2795     __ j(Done);
2796   }
2797 
2798   __ bind(notObj);
2799   __ subi(t0, tos_state, (u1)itos);
2800   __ bnez(t0, notInt);
2801 
2802   // itos
2803   {
2804     __ pop(itos);
2805     // field address
2806     if (!is_static) {
2807       pop_and_check_object(obj);
2808     }
2809     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2810     const Address field(off, 0);
2811     __ access_store_at(T_INT, IN_HEAP, field, x10, noreg, noreg, noreg);
2812     if (rc == may_rewrite) {
2813       patch_bytecode(Bytecodes::_fast_iputfield, bc, x11, true, byte_no);
2814     }
2815     __ j(Done);
2816   }
2817 
2818   __ bind(notInt);
2819   __ subi(t0, tos_state, (u1)ctos);
2820   __ bnez(t0, notChar);
2821 
2822   // ctos
2823   {
2824     __ pop(ctos);
2825     // field address
2826     if (!is_static) {
2827       pop_and_check_object(obj);
2828     }
2829     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2830     const Address field(off, 0);
2831     __ access_store_at(T_CHAR, IN_HEAP, field, x10, noreg, noreg, noreg);
2832     if (rc == may_rewrite) {
2833       patch_bytecode(Bytecodes::_fast_cputfield, bc, x11, true, byte_no);
2834     }
2835     __ j(Done);
2836   }
2837 
2838   __ bind(notChar);
2839   __ subi(t0, tos_state, (u1)stos);
2840   __ bnez(t0, notShort);
2841 
2842   // stos
2843   {
2844     __ pop(stos);
2845     // field address
2846     if (!is_static) {
2847       pop_and_check_object(obj);
2848     }
2849     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2850     const Address field(off, 0);
2851     __ access_store_at(T_SHORT, IN_HEAP, field, x10, noreg, noreg, noreg);
2852     if (rc == may_rewrite) {
2853       patch_bytecode(Bytecodes::_fast_sputfield, bc, x11, true, byte_no);
2854     }
2855     __ j(Done);
2856   }
2857 
2858   __ bind(notShort);
2859   __ subi(t0, tos_state, (u1)ltos);
2860   __ bnez(t0, notLong);
2861 
2862   // ltos
2863   {
2864     __ pop(ltos);
2865     // field address
2866     if (!is_static) {
2867       pop_and_check_object(obj);
2868     }
2869     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2870     const Address field(off, 0);
2871     __ access_store_at(T_LONG, IN_HEAP, field, x10, noreg, noreg, noreg);
2872     if (rc == may_rewrite) {
2873       patch_bytecode(Bytecodes::_fast_lputfield, bc, x11, true, byte_no);
2874     }
2875     __ j(Done);
2876   }
2877 
2878   __ bind(notLong);
2879   __ subi(t0, tos_state, (u1)ftos);
2880   __ bnez(t0, notFloat);
2881 
2882   // ftos
2883   {
2884     __ pop(ftos);
2885     // field address
2886     if (!is_static) {
2887       pop_and_check_object(obj);
2888     }
2889     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2890     const Address field(off, 0);
2891     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg, noreg);
2892     if (rc == may_rewrite) {
2893       patch_bytecode(Bytecodes::_fast_fputfield, bc, x11, true, byte_no);
2894     }
2895     __ j(Done);
2896   }
2897 
2898   __ bind(notFloat);
2899 #ifdef ASSERT
2900   __ subi(t0, tos_state, (u1)dtos);
2901   __ bnez(t0, notDouble);
2902 #endif
2903 
2904   // dtos
2905   {
2906     __ pop(dtos);
2907     // field address
2908     if (!is_static) {
2909       pop_and_check_object(obj);
2910     }
2911     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2912     const Address field(off, 0);
2913     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg, noreg);
2914     if (rc == may_rewrite) {
2915       patch_bytecode(Bytecodes::_fast_dputfield, bc, x11, true, byte_no);
2916     }
2917   }
2918 
2919 #ifdef ASSERT
2920   __ j(Done);
2921 
2922   __ bind(notDouble);
2923   __ stop("Bad state");
2924 #endif
2925 
2926   __ bind(Done);
2927 
2928   {
2929     Label notVolatile;
2930     __ test_bit(t0, x15, ResolvedFieldEntry::is_volatile_shift);
2931     __ beqz(t0, notVolatile);
2932     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
2933     __ bind(notVolatile);
2934   }
2935 }
2936 
2937 void TemplateTable::putfield(int byte_no) {
2938   putfield_or_static(byte_no, false);
2939 }
2940 
2941 void TemplateTable::nofast_putfield(int byte_no) {
2942   putfield_or_static(byte_no, false, may_not_rewrite);
2943 }
2944 
2945 void TemplateTable::putstatic(int byte_no) {
2946   putfield_or_static(byte_no, true);
2947 }
2948 
2949 void TemplateTable::jvmti_post_fast_field_mod() {
2950   if (JvmtiExport::can_post_field_modification()) {
2951     // Check to see if a field modification watch has been set before
2952     // we take the time to call into the VM.
2953     Label L2;
2954     __ lwu(c_rarg3, ExternalAddress(JvmtiExport::get_field_modification_count_addr()));
2955     __ beqz(c_rarg3, L2);
2956 
2957     __ pop_ptr(x9);                  // copy the object pointer from tos
2958     __ verify_oop(x9);
2959     __ push_ptr(x9);                 // put the object pointer back on tos
2960     // Save tos values before call_VM() clobbers them. Since we have
2961     // to do it for every data type, we use the saved values as the
2962     // jvalue object.
2963     switch (bytecode()) {          // load values into the jvalue object
2964       case Bytecodes::_fast_aputfield: __ push_ptr(x10); break;
2965       case Bytecodes::_fast_bputfield: // fall through
2966       case Bytecodes::_fast_zputfield: // fall through
2967       case Bytecodes::_fast_sputfield: // fall through
2968       case Bytecodes::_fast_cputfield: // fall through
2969       case Bytecodes::_fast_iputfield: __ push_i(x10); break;
2970       case Bytecodes::_fast_dputfield: __ push_d(); break;
2971       case Bytecodes::_fast_fputfield: __ push_f(); break;
2972       case Bytecodes::_fast_lputfield: __ push_l(x10); break;
2973 
2974       default:
2975         ShouldNotReachHere();
2976     }
2977     __ mv(c_rarg3, esp);             // points to jvalue on the stack
2978     // access constant pool cache entry
2979     __ load_field_entry(c_rarg2, x10);
2980     __ verify_oop(x9);
2981     // x9: object pointer copied above
2982     // c_rarg2: cache entry pointer
2983     // c_rarg3: jvalue object on the stack
2984     __ call_VM(noreg,
2985                CAST_FROM_FN_PTR(address,
2986                                 InterpreterRuntime::post_field_modification),
2987                x9, c_rarg2, c_rarg3);
2988 
2989     switch (bytecode()) {             // restore tos values
2990       case Bytecodes::_fast_aputfield: __ pop_ptr(x10); break;
2991       case Bytecodes::_fast_bputfield: // fall through
2992       case Bytecodes::_fast_zputfield: // fall through
2993       case Bytecodes::_fast_sputfield: // fall through
2994       case Bytecodes::_fast_cputfield: // fall through
2995       case Bytecodes::_fast_iputfield: __ pop_i(x10); break;
2996       case Bytecodes::_fast_dputfield: __ pop_d(); break;
2997       case Bytecodes::_fast_fputfield: __ pop_f(); break;
2998       case Bytecodes::_fast_lputfield: __ pop_l(x10); break;
2999       default: break;
3000     }
3001     __ bind(L2);
3002   }
3003 }
3004 
3005 void TemplateTable::fast_storefield(TosState state) {
3006   transition(state, vtos);
3007 
3008   ByteSize base = ConstantPoolCache::base_offset();
3009 
3010   jvmti_post_fast_field_mod();
3011 
3012   // access constant pool cache
3013   __ load_field_entry(x12, x11);
3014 
3015   // X11: field offset, X12: field holder, X13: flags
3016   load_resolved_field_entry(x12, x12, noreg, x11, x13);
3017 
3018   {
3019     Label notVolatile;
3020     __ test_bit(t0, x13, ResolvedFieldEntry::is_volatile_shift);
3021     __ beqz(t0, notVolatile);
3022     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
3023     __ bind(notVolatile);
3024   }
3025 
3026   // Get object from stack
3027   pop_and_check_object(x12);
3028 
3029   // field address
3030   __ add(x11, x12, x11);
3031   const Address field(x11, 0);
3032 
3033   // access field, must not clobber x13 - flags
3034   switch (bytecode()) {
3035     case Bytecodes::_fast_aputfield:
3036       __ store_heap_oop(field, x10, x28, x29, x15, IN_HEAP);
3037       break;
3038     case Bytecodes::_fast_lputfield:
3039       __ access_store_at(T_LONG, IN_HEAP, field, x10, noreg, noreg, noreg);
3040       break;
3041     case Bytecodes::_fast_iputfield:
3042       __ access_store_at(T_INT, IN_HEAP, field, x10, noreg, noreg, noreg);
3043       break;
3044     case Bytecodes::_fast_zputfield:
3045       __ access_store_at(T_BOOLEAN, IN_HEAP, field, x10, noreg, noreg, noreg);
3046       break;
3047     case Bytecodes::_fast_bputfield:
3048       __ access_store_at(T_BYTE, IN_HEAP, field, x10, noreg, noreg, noreg);
3049       break;
3050     case Bytecodes::_fast_sputfield:
3051       __ access_store_at(T_SHORT, IN_HEAP, field, x10, noreg, noreg, noreg);
3052       break;
3053     case Bytecodes::_fast_cputfield:
3054       __ access_store_at(T_CHAR, IN_HEAP, field, x10, noreg, noreg, noreg);
3055       break;
3056     case Bytecodes::_fast_fputfield:
3057       __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg, noreg);
3058       break;
3059     case Bytecodes::_fast_dputfield:
3060       __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg, noreg);
3061       break;
3062     default:
3063       ShouldNotReachHere();
3064   }
3065 
3066   {
3067     Label notVolatile;
3068     __ test_bit(t0, x13, ResolvedFieldEntry::is_volatile_shift);
3069     __ beqz(t0, notVolatile);
3070     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
3071     __ bind(notVolatile);
3072   }
3073 }
3074 
3075 void TemplateTable::fast_accessfield(TosState state) {
3076   transition(atos, state);
3077   // Do the JVMTI work here to avoid disturbing the register state below
3078   if (JvmtiExport::can_post_field_access()) {
3079     // Check to see if a field access watch has been set before we
3080     // take the time to call into the VM.
3081     Label L1;
3082     __ lwu(x12, ExternalAddress(JvmtiExport::get_field_access_count_addr()));
3083     __ beqz(x12, L1);
3084 
3085     // access constant pool cache entry
3086     __ load_field_entry(c_rarg2, t1);
3087     __ verify_oop(x10);
3088     __ push_ptr(x10);  // save object pointer before call_VM() clobbers it
3089     __ mv(c_rarg1, x10);
3090     // c_rarg1: object pointer copied above
3091     // c_rarg2: cache entry pointer
3092     __ call_VM(noreg,
3093                CAST_FROM_FN_PTR(address,
3094                                 InterpreterRuntime::post_field_access),
3095                c_rarg1, c_rarg2);
3096     __ pop_ptr(x10); // restore object pointer
3097     __ bind(L1);
3098   }
3099 
3100   // access constant pool cache
3101   __ load_field_entry(x12, x11);
3102 
3103   __ load_sized_value(x11, Address(x12, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
3104   __ load_unsigned_byte(x13, Address(x12, in_bytes(ResolvedFieldEntry::flags_offset())));
3105 
3106   // x10: object
3107   __ verify_oop(x10);
3108   __ null_check(x10);
3109   __ add(x11, x10, x11);
3110   const Address field(x11, 0);
3111 
3112   // access field
3113   switch (bytecode()) {
3114     case Bytecodes::_fast_agetfield:
3115       __ load_heap_oop(x10, field, x28, x29, IN_HEAP);
3116       __ verify_oop(x10);
3117       break;
3118     case Bytecodes::_fast_lgetfield:
3119       __ access_load_at(T_LONG, IN_HEAP, x10, field, noreg, noreg);
3120       break;
3121     case Bytecodes::_fast_igetfield:
3122       __ access_load_at(T_INT, IN_HEAP, x10, field, noreg, noreg);
3123       __ sext(x10, x10, 32);
3124       break;
3125     case Bytecodes::_fast_bgetfield:
3126       __ access_load_at(T_BYTE, IN_HEAP, x10, field, noreg, noreg);
3127       break;
3128     case Bytecodes::_fast_sgetfield:
3129       __ access_load_at(T_SHORT, IN_HEAP, x10, field, noreg, noreg);
3130       break;
3131     case Bytecodes::_fast_cgetfield:
3132       __ access_load_at(T_CHAR, IN_HEAP, x10, field, noreg, noreg);
3133       break;
3134     case Bytecodes::_fast_fgetfield:
3135       __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
3136       break;
3137     case Bytecodes::_fast_dgetfield:
3138       __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* dtos */, field, noreg, noreg);
3139       break;
3140     default:
3141       ShouldNotReachHere();
3142   }
3143   {
3144     Label notVolatile;
3145     __ test_bit(t0, x13, ResolvedFieldEntry::is_volatile_shift);
3146     __ beqz(t0, notVolatile);
3147     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3148     __ bind(notVolatile);
3149   }
3150 }
3151 
3152 void TemplateTable::fast_xaccess(TosState state) {
3153   transition(vtos, state);
3154 
3155   // get receiver
3156   __ ld(x10, aaddress(0));
3157   // access constant pool cache
3158   __ load_field_entry(x12, x13, 2);
3159   __ load_sized_value(x11, Address(x12, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
3160 
3161   // make sure exception is reported in correct bcp range (getfield is
3162   // next instruction)
3163   __ addi(xbcp, xbcp, 1);
3164   __ null_check(x10);
3165   switch (state) {
3166     case itos:
3167       __ add(x10, x10, x11);
3168       __ access_load_at(T_INT, IN_HEAP, x10, Address(x10, 0), noreg, noreg);
3169       __ sext(x10, x10, 32);
3170       break;
3171     case atos:
3172       __ add(x10, x10, x11);
3173       __ load_heap_oop(x10, Address(x10, 0), x28, x29, IN_HEAP);
3174       __ verify_oop(x10);
3175       break;
3176     case ftos:
3177       __ add(x10, x10, x11);
3178       __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, Address(x10), noreg, noreg);
3179       break;
3180     default:
3181       ShouldNotReachHere();
3182   }
3183 
3184   {
3185     Label notVolatile;
3186     __ load_unsigned_byte(x13, Address(x12, in_bytes(ResolvedFieldEntry::flags_offset())));
3187     __ test_bit(t0, x13, ResolvedFieldEntry::is_volatile_shift);
3188     __ beqz(t0, notVolatile);
3189     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3190     __ bind(notVolatile);
3191   }
3192 
3193   __ subi(xbcp, xbcp, 1);
3194 }
3195 
3196 //-----------------------------------------------------------------------------
3197 // Calls
3198 
3199 void TemplateTable::prepare_invoke(Register cache, Register recv) {
3200 
3201   Bytecodes::Code code = bytecode();
3202   const bool load_receiver       = (code != Bytecodes::_invokestatic) && (code != Bytecodes::_invokedynamic);
3203 
3204   // save 'interpreter return address'
3205   __ save_bcp();
3206 
3207   // Load TOS state for later
3208   __ load_unsigned_byte(t1, Address(cache, in_bytes(ResolvedMethodEntry::type_offset())));
3209 
3210   // load receiver if needed (note: no return address pushed yet)
3211   if (load_receiver) {
3212     __ load_unsigned_short(recv, Address(cache, in_bytes(ResolvedMethodEntry::num_parameters_offset())));
3213     __ shadd(t0, recv, esp, t0, 3);
3214     __ ld(recv, Address(t0, -Interpreter::expr_offset_in_bytes(1)));
3215     __ verify_oop(recv);
3216   }
3217 
3218   // load return address
3219   {
3220     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3221     __ mv(t0, table_addr);
3222     __ shadd(t0, t1, t0, t1, 3);
3223     __ ld(ra, Address(t0, 0));
3224   }
3225 }
3226 
3227 void TemplateTable::invokevirtual_helper(Register index,
3228                                          Register recv,
3229                                          Register flags) {
3230   // Uses temporary registers x10, x13
3231   assert_different_registers(index, recv, x10, x13);
3232   // Test for an invoke of a final method
3233   Label notFinal;
3234   __ test_bit(t0, flags, ResolvedMethodEntry::is_vfinal_shift);
3235   __ beqz(t0, notFinal);
3236 
3237   const Register method = index;  // method must be xmethod
3238   assert(method == xmethod, "Method must be xmethod for interpreter calling convention");
3239 
3240   // do the call - the index is actually the method to call
3241   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3242 
3243   // It's final, need a null check here!
3244   __ null_check(recv);
3245 
3246   // profile this call
3247   __ profile_final_call(x10);
3248   __ profile_arguments_type(x10, method, x14, true);
3249 
3250   __ jump_from_interpreted(method);
3251 
3252   __ bind(notFinal);
3253 
3254   // get receiver klass
3255   __ load_klass(x10, recv);
3256 
3257   // profile this call
3258   __ profile_virtual_call(x10, xlocals, x13);
3259 
3260   // get target Method & entry point
3261   __ lookup_virtual_method(x10, index, method);
3262   __ profile_arguments_type(x13, method, x14, true);
3263   __ jump_from_interpreted(method);
3264 }
3265 
3266 void TemplateTable::invokevirtual(int byte_no) {
3267   transition(vtos, vtos);
3268   assert(byte_no == f2_byte, "use this argument");
3269 
3270   load_resolved_method_entry_virtual(x12,      // ResolvedMethodEntry*
3271                                      xmethod, // Method* or itable index
3272                                      x13);     // flags
3273   prepare_invoke(x12, x12); // recv
3274 
3275   // xmethod: index (actually a Method*)
3276   // x12: receiver
3277   // x13: flags
3278 
3279   invokevirtual_helper(xmethod, x12, x13);
3280 }
3281 
3282 void TemplateTable::invokespecial(int byte_no) {
3283   transition(vtos, vtos);
3284   assert(byte_no == f1_byte, "use this argument");
3285 
3286   load_resolved_method_entry_special_or_static(x12,      // ResolvedMethodEntry*
3287                                                xmethod, // Method*
3288                                                x13);     // flags
3289   prepare_invoke(x12, x12);  // get receiver also for null check
3290 
3291   __ verify_oop(x12);
3292   __ null_check(x12);
3293   // do the call
3294   __ profile_call(x10);
3295   __ profile_arguments_type(x10, xmethod, xbcp, false);
3296   __ jump_from_interpreted(xmethod);
3297 }
3298 
3299 void TemplateTable::invokestatic(int byte_no) {
3300   transition(vtos, vtos);
3301   assert(byte_no == f1_byte, "use this argument");
3302 
3303   load_resolved_method_entry_special_or_static(x12,      // ResolvedMethodEntry*
3304                                                xmethod, // Method*
3305                                                x13);     // flags
3306   prepare_invoke(x12, x12);  // get receiver also for null check
3307 
3308   // do the call
3309   __ profile_call(x10);
3310   __ profile_arguments_type(x10, xmethod, x14, false);
3311   __ jump_from_interpreted(xmethod);
3312 }
3313 
3314 void TemplateTable::fast_invokevfinal(int byte_no) {
3315   __ call_Unimplemented();
3316 }
3317 
3318 void TemplateTable::invokeinterface(int byte_no) {
3319   transition(vtos, vtos);
3320   assert(byte_no == f1_byte, "use this argument");
3321 
3322   load_resolved_method_entry_interface(x12,      // ResolvedMethodEntry*
3323                                        x10,      // Klass*
3324                                        xmethod, // Method* or itable/vtable index
3325                                        x13);     // flags
3326   prepare_invoke(x12, x12); // receiver
3327 
3328   // x10: interface klass (from f1)
3329   // xmethod: method (from f2)
3330   // x12: receiver
3331   // x13: flags
3332 
3333   // First check for Object case, then private interface method,
3334   // then regular interface method.
3335 
3336   // Special case of invokeinterface called for virtual method of
3337   // java.lang.Object. See cpCache.cpp for details
3338   Label notObjectMethod;
3339   __ test_bit(t0, x13, ResolvedMethodEntry::is_forced_virtual_shift);
3340   __ beqz(t0, notObjectMethod);
3341 
3342   invokevirtual_helper(xmethod, x12, x13);
3343   __ bind(notObjectMethod);
3344 
3345   Label no_such_interface;
3346 
3347   // Check for private method invocation - indicated by vfinal
3348   Label notVFinal;
3349   __ test_bit(t0, x13, ResolvedMethodEntry::is_vfinal_shift);
3350   __ beqz(t0, notVFinal);
3351 
3352   // Check receiver klass into x13
3353   __ load_klass(x13, x12);
3354 
3355   Label subtype;
3356   __ check_klass_subtype(x13, x10, x14, subtype);
3357   // If we get here the typecheck failed
3358   __ j(no_such_interface);
3359   __ bind(subtype);
3360 
3361   __ profile_final_call(x10);
3362   __ profile_arguments_type(x10, xmethod, x14, true);
3363   __ jump_from_interpreted(xmethod);
3364 
3365   __ bind(notVFinal);
3366 
3367   // Get receiver klass into x13
3368   __ restore_locals();
3369   __ load_klass(x13, x12);
3370 
3371   Label no_such_method;
3372 
3373   // Preserve method for the throw_AbstractMethodErrorVerbose.
3374   __ mv(x28, xmethod);
3375   // Receiver subtype check against REFC.
3376   // Superklass in x10. Subklass in x13. Blows t1, x30
3377   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3378                              x13, x10, noreg,
3379                              // outputs: scan temp. reg, scan temp. reg
3380                              t1, x30,
3381                              no_such_interface,
3382                              /*return_method=*/false);
3383 
3384   // profile this call
3385   __ profile_virtual_call(x13, x30, x9);
3386 
3387   // Get declaring interface class from method, and itable index
3388   __ load_method_holder(x10, xmethod);
3389   __ lwu(xmethod, Address(xmethod, Method::itable_index_offset()));
3390   __ subw(xmethod, xmethod, Method::itable_index_max);
3391   __ negw(xmethod, xmethod);
3392 
3393   // Preserve recvKlass for throw_AbstractMethodErrorVerbose
3394   __ mv(xlocals, x13);
3395   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3396                              xlocals, x10, xmethod,
3397                              // outputs: method, scan temp. reg
3398                              xmethod, x30,
3399                              no_such_interface);
3400 
3401   // xmethod: Method to call
3402   // x12: receiver
3403   // Check for abstract method error
3404   // Note: This should be done more efficiently via a throw_abstract_method_error
3405   //       interpreter entry point and a conditional jump to it in case of a null
3406   //       method.
3407   __ beqz(xmethod, no_such_method);
3408 
3409   __ profile_arguments_type(x13, xmethod, x30, true);
3410 
3411   // do the call
3412   // x12: receiver
3413   // xmethod: Method
3414   __ jump_from_interpreted(xmethod);
3415   __ should_not_reach_here();
3416 
3417   // exception handling code follows ...
3418   // note: must restore interpreter registers to canonical
3419   //       state for exception handling to work correctly!
3420 
3421   __ bind(no_such_method);
3422   // throw exception
3423   __ restore_bcp();    // bcp must be correct for exception handler   (was destroyed)
3424   __ restore_locals(); // make sure locals pointer is correct as well (was destroyed)
3425   // Pass arguments for generating a verbose error message.
3426   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodErrorVerbose), x13, x28);
3427   // the call_VM checks for exception, so we should never return here.
3428   __ should_not_reach_here();
3429 
3430   __ bind(no_such_interface);
3431   // throw exceptiong
3432   __ restore_bcp();    // bcp must be correct for exception handler   (was destroyed)
3433   __ restore_locals(); // make sure locals pointer is correct as well (was destroyed)
3434   // Pass arguments for generating a verbose error message.
3435   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3436                                      InterpreterRuntime::throw_IncompatibleClassChangeErrorVerbose), x13, x10);
3437   // the call_VM checks for exception, so we should never return here.
3438   __ should_not_reach_here();
3439   return;
3440 }
3441 
3442 void TemplateTable::invokehandle(int byte_no) {
3443   transition(vtos, vtos);
3444   assert(byte_no == f1_byte, "use this argument");
3445 
3446   load_resolved_method_entry_handle(x12,      // ResolvedMethodEntry*
3447                                     xmethod, // Method*
3448                                     x10,      // Resolved reference
3449                                     x13);     // flags
3450   prepare_invoke(x12, x12);
3451 
3452   __ verify_method_ptr(x12);
3453   __ verify_oop(x12);
3454   __ null_check(x12);
3455 
3456   // FIXME: profile the LambdaForm also
3457 
3458   // x30 is safe to use here as a temp reg because it is about to
3459   // be clobbered by jump_from_interpreted().
3460   __ profile_final_call(x30);
3461   __ profile_arguments_type(x30, xmethod, x14, true);
3462 
3463   __ jump_from_interpreted(xmethod);
3464 }
3465 
3466 void TemplateTable::invokedynamic(int byte_no) {
3467   transition(vtos, vtos);
3468   assert(byte_no == f1_byte, "use this argument");
3469 
3470   load_invokedynamic_entry(xmethod);
3471 
3472   // x10: CallSite object (from cpool->resolved_references[])
3473   // xmethod: MH.linkToCallSite method
3474 
3475   // Note: x10_callsite is already pushed
3476 
3477   // %%% should make a type profile for any invokedynamic that takes a ref argument
3478   // profile this call
3479   __ profile_call(xbcp);
3480   __ profile_arguments_type(x13, xmethod, x30, false);
3481 
3482   __ verify_oop(x10);
3483 
3484   __ jump_from_interpreted(xmethod);
3485 }
3486 
3487 //-----------------------------------------------------------------------------
3488 // Allocation
3489 
3490 void TemplateTable::_new() {
3491   transition(vtos, atos);
3492 
3493   __ get_unsigned_2_byte_index_at_bcp(x13, 1);
3494   Label slow_case;
3495   Label done;
3496   Label initialize_header;
3497 
3498   __ get_cpool_and_tags(x14, x10);
3499   // Make sure the class we're about to instantiate has been resolved.
3500   // This is done before loading InstanceKlass to be consistent with the order
3501   // how Constant Pool is update (see ConstantPool::klass_at_put)
3502   const int tags_offset = Array<u1>::base_offset_in_bytes();
3503   __ add(t0, x10, x13);
3504   __ la(t0, Address(t0, tags_offset));
3505   __ lbu(t0, t0);
3506   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3507   __ subi(t1, t0, (u1)JVM_CONSTANT_Class);
3508   __ bnez(t1, slow_case);
3509 
3510   // get InstanceKlass
3511   __ load_resolved_klass_at_offset(x14, x13, x14, t0);
3512 
3513   // make sure klass is initialized
3514   assert(VM_Version::supports_fast_class_init_checks(),
3515          "Optimization requires support for fast class initialization checks");
3516   __ clinit_barrier(x14, t0, nullptr /*L_fast_path*/, &slow_case);
3517 
3518   // get instance_size in InstanceKlass (scaled to a count of bytes)
3519   __ lwu(x13, Address(x14, Klass::layout_helper_offset()));
3520   // test to see if is malformed in some way
3521   __ test_bit(t0, x13, exact_log2(Klass::_lh_instance_slow_path_bit));
3522   __ bnez(t0, slow_case);
3523 
3524   // Allocate the instance:
3525   //  If TLAB is enabled:
3526   //    Try to allocate in the TLAB.
3527   //    If fails, go to the slow path.
3528   //    Initialize the allocation.
3529   //    Exit.
3530   //  Go to slow path.
3531 
3532   if (UseTLAB) {
3533     __ tlab_allocate(x10, x13, 0, noreg, x11, slow_case);
3534 
3535     if (ZeroTLAB) {
3536       // the fields have been already cleared
3537       __ j(initialize_header);
3538     }
3539 
3540     // The object is initialized before the header. If the object size is
3541     // zero, go directly to the header initialization.
3542     if (UseCompactObjectHeaders) {
3543       assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
3544       __ subi(x13, x13, oopDesc::base_offset_in_bytes());
3545     } else {
3546       __ subi(x13, x13, sizeof(oopDesc));
3547     }
3548     __ beqz(x13, initialize_header);
3549 
3550     // Initialize object fields
3551     {
3552       if (UseCompactObjectHeaders) {
3553         assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
3554         __ addi(x12, x10, oopDesc::base_offset_in_bytes());
3555       } else {
3556         __ addi(x12, x10, sizeof(oopDesc));
3557       }
3558       Label loop;
3559       __ bind(loop);
3560       __ sd(zr, Address(x12));
3561       __ addi(x12, x12, BytesPerLong);
3562       __ subi(x13, x13, BytesPerLong);
3563       __ bnez(x13, loop);
3564     }
3565 
3566     // initialize object hader only.
3567     __ bind(initialize_header);
3568     if (UseCompactObjectHeaders) {
3569       __ ld(t0, Address(x14, Klass::prototype_header_offset()));
3570       __ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes()));
3571     } else {
3572       __ mv(t0, (intptr_t)markWord::prototype().value());
3573       __ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes()));
3574       __ store_klass_gap(x10, zr);   // zero klass gap for compressed oops
3575       __ store_klass(x10, x14);      // store klass last
3576     }
3577 
3578     if (DTraceAllocProbes) {
3579       // Trigger dtrace event for fastpath
3580       __ push(atos); // save the return value
3581       __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), x10);
3582       __ pop(atos); // restore the return value
3583     }
3584     __ j(done);
3585   }
3586 
3587   // slow case
3588   __ bind(slow_case);
3589   __ get_constant_pool(c_rarg1);
3590   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3591   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3592   __ verify_oop(x10);
3593 
3594   // continue
3595   __ bind(done);
3596   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3597   __ membar(MacroAssembler::StoreStore);
3598 }
3599 
3600 void TemplateTable::newarray() {
3601   transition(itos, atos);
3602   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3603   __ mv(c_rarg2, x10);
3604   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3605           c_rarg1, c_rarg2);
3606   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3607   __ membar(MacroAssembler::StoreStore);
3608 }
3609 
3610 void TemplateTable::anewarray() {
3611   transition(itos, atos);
3612   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3613   __ get_constant_pool(c_rarg1);
3614   __ mv(c_rarg3, x10);
3615   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3616           c_rarg1, c_rarg2, c_rarg3);
3617   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3618   __ membar(MacroAssembler::StoreStore);
3619 }
3620 
3621 void TemplateTable::arraylength() {
3622   transition(atos, itos);
3623   __ lwu(x10, Address(x10, arrayOopDesc::length_offset_in_bytes()));
3624 }
3625 
3626 void TemplateTable::checkcast() {
3627   transition(atos, atos);
3628   Label done, is_null, ok_is_subtype, quicked, resolved;
3629   __ beqz(x10, is_null);
3630 
3631   // Get cpool & tags index
3632   __ get_cpool_and_tags(x12, x13); // x12=cpool, x13=tags array
3633   __ get_unsigned_2_byte_index_at_bcp(x9, 1); // x9=index
3634   // See if bytecode has already been quicked
3635   __ addi(t0, x13, Array<u1>::base_offset_in_bytes());
3636   __ add(x11, t0, x9);
3637   __ lbu(x11, x11);
3638   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3639   __ subi(t0, x11, (u1)JVM_CONSTANT_Class);
3640   __ beqz(t0, quicked);
3641 
3642   __ push(atos); // save receiver for result, and for GC
3643   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3644   __ get_vm_result_metadata(x10, xthread);
3645   __ pop_reg(x13); // restore receiver
3646   __ j(resolved);
3647 
3648   // Get superklass in x10 and subklass in x13
3649   __ bind(quicked);
3650   __ mv(x13, x10); // Save object in x13; x10 needed for subtype check
3651   __ load_resolved_klass_at_offset(x12, x9, x10, t0); // x10 = klass
3652 
3653   __ bind(resolved);
3654   __ load_klass(x9, x13);
3655 
3656   // Generate subtype check.  Blows x12, x15.  Object in x13.
3657   // Superklass in x10.  Subklass in x9.
3658   __ gen_subtype_check(x9, ok_is_subtype);
3659 
3660   // Come here on failure
3661   __ push_reg(x13);
3662   // object is at TOS
3663   __ j(RuntimeAddress(Interpreter::_throw_ClassCastException_entry));
3664 
3665   // Come here on success
3666   __ bind(ok_is_subtype);
3667   __ mv(x10, x13); // Restore object in x13
3668 
3669   // Collect counts on whether this test sees nulls a lot or not.
3670   if (ProfileInterpreter) {
3671     __ j(done);
3672     __ bind(is_null);
3673     __ profile_null_seen(x12);
3674   } else {
3675     __ bind(is_null);   // same as 'done'
3676   }
3677   __ bind(done);
3678 }
3679 
3680 void TemplateTable::instanceof() {
3681   transition(atos, itos);
3682   Label done, is_null, ok_is_subtype, quicked, resolved;
3683   __ beqz(x10, is_null);
3684 
3685   // Get cpool & tags index
3686   __ get_cpool_and_tags(x12, x13); // x12=cpool, x13=tags array
3687   __ get_unsigned_2_byte_index_at_bcp(x9, 1); // x9=index
3688   // See if bytecode has already been quicked
3689   __ addi(t0, x13, Array<u1>::base_offset_in_bytes());
3690   __ add(x11, t0, x9);
3691   __ lbu(x11, x11);
3692   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3693   __ subi(t0, x11, (u1)JVM_CONSTANT_Class);
3694   __ beqz(t0, quicked);
3695 
3696   __ push(atos); // save receiver for result, and for GC
3697   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3698   __ get_vm_result_metadata(x10, xthread);
3699   __ pop_reg(x13); // restore receiver
3700   __ verify_oop(x13);
3701   __ load_klass(x13, x13);
3702   __ j(resolved);
3703 
3704   // Get superklass in x10 and subklass in x13
3705   __ bind(quicked);
3706   __ load_klass(x13, x10);
3707   __ load_resolved_klass_at_offset(x12, x9, x10, t0);
3708 
3709   __ bind(resolved);
3710 
3711   // Generate subtype check.  Blows x12, x15
3712   // Superklass in x10.  Subklass in x13.
3713   __ gen_subtype_check(x13, ok_is_subtype);
3714 
3715   // Come here on failure
3716   __ mv(x10, zr);
3717   __ j(done);
3718   // Come here on success
3719   __ bind(ok_is_subtype);
3720   __ mv(x10, 1);
3721 
3722   // Collect counts on whether this test sees nulls a lot or not.
3723   if (ProfileInterpreter) {
3724     __ j(done);
3725     __ bind(is_null);
3726     __ profile_null_seen(x12);
3727   } else {
3728     __ bind(is_null);   // same as 'done'
3729   }
3730   __ bind(done);
3731   // x10 = 0: obj is    null or  obj is not an instanceof the specified klass
3732   // x10 = 1: obj isn't null and obj is     an instanceof the specified klass
3733 }
3734 
3735 //-----------------------------------------------------------------------------
3736 // Breakpoints
3737 
3738 void TemplateTable::_breakpoint() {
3739   // Note: We get here even if we are single stepping..
3740   // jbug inists on setting breakpoints at every bytecode
3741   // even if we are in single step mode.
3742 
3743   transition(vtos, vtos);
3744 
3745   // get the unpatched byte code
3746   __ get_method(c_rarg1);
3747   __ call_VM(noreg,
3748              CAST_FROM_FN_PTR(address,
3749                               InterpreterRuntime::get_original_bytecode_at),
3750              c_rarg1, xbcp);
3751   __ mv(x9, x10);
3752 
3753   // post the breakpoint event
3754   __ call_VM(noreg,
3755              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
3756              xmethod, xbcp);
3757 
3758   // complete the execution of original bytecode
3759   __ mv(t0, x9);
3760   __ dispatch_only_normal(vtos);
3761 }
3762 
3763 //-----------------------------------------------------------------------------
3764 // Exceptions
3765 
3766 void TemplateTable::athrow() {
3767   transition(atos, vtos);
3768   __ null_check(x10);
3769   __ j(RuntimeAddress(Interpreter::throw_exception_entry()));
3770 }
3771 
3772 //-----------------------------------------------------------------------------
3773 // Synchronization
3774 //
3775 // Note: monitorenter & exit are symmetric routines; which is reflected
3776 //       in the assembly code structure as well
3777 //
3778 // Stack layout:
3779 //
3780 // [expressions  ] <--- esp               = expression stack top
3781 // ..
3782 // [expressions  ]
3783 // [monitor entry] <--- monitor block top = expression stack bot
3784 // ..
3785 // [monitor entry]
3786 // [frame data   ] <--- monitor block bot
3787 // ...
3788 // [saved fp     ] <--- fp
3789 
3790 void TemplateTable::monitorenter() {
3791   transition(atos, vtos);
3792 
3793    // check for null object
3794    __ null_check(x10);
3795 
3796    const Address monitor_block_top(
3797          fp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3798    const Address monitor_block_bot(
3799          fp, frame::interpreter_frame_initial_sp_offset * wordSize);
3800    const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
3801 
3802    Label allocated;
3803 
3804    // initialize entry pointer
3805    __ mv(c_rarg1, zr); // points to free slot or null
3806 
3807    // find a free slot in the monitor block (result in c_rarg1)
3808    {
3809      Label entry, loop, exit, notUsed;
3810      __ ld(c_rarg3, monitor_block_top); // derelativize pointer
3811      __ shadd(c_rarg3, c_rarg3, fp, c_rarg3, LogBytesPerWord);
3812      // Now c_rarg3 points to current entry, starting with top-most entry
3813 
3814      __ la(c_rarg2, monitor_block_bot); // points to word before bottom
3815 
3816      __ j(entry);
3817 
3818      __ bind(loop);
3819      // check if current entry is used
3820      // if not used then remember entry in c_rarg1
3821      __ ld(t0, Address(c_rarg3, BasicObjectLock::obj_offset()));
3822      __ bnez(t0, notUsed);
3823      __ mv(c_rarg1, c_rarg3);
3824      __ bind(notUsed);
3825      // check if current entry is for same object
3826      // if same object then stop searching
3827      __ beq(x10, t0, exit);
3828      // otherwise advance to next entry
3829      __ add(c_rarg3, c_rarg3, entry_size);
3830      __ bind(entry);
3831      // check if bottom reached
3832      // if not at bottom then check this entry
3833      __ bne(c_rarg3, c_rarg2, loop);
3834      __ bind(exit);
3835    }
3836 
3837    __ bnez(c_rarg1, allocated); // check if a slot has been found and
3838                              // if found, continue with that on
3839 
3840    // allocate one if there's no free slot
3841    {
3842      Label entry, loop;
3843      // 1. compute new pointers            // esp: old expression stack top
3844 
3845      __ check_extended_sp();
3846      __ sub(sp, sp, entry_size);           // make room for the monitor
3847      __ sub(t0, sp, fp);
3848      __ srai(t0, t0, Interpreter::logStackElementSize);
3849      __ sd(t0, Address(fp, frame::interpreter_frame_extended_sp_offset * wordSize));
3850 
3851      __ ld(c_rarg1, monitor_block_bot);    // derelativize pointer
3852      __ shadd(c_rarg1, c_rarg1, fp, c_rarg1, LogBytesPerWord);
3853      // Now c_rarg1 points to the old expression stack bottom
3854 
3855      __ sub(esp, esp, entry_size);         // move expression stack top
3856      __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
3857      __ mv(c_rarg3, esp);                  // set start value for copy loop
3858      __ sub(t0, c_rarg1, fp);              // relativize pointer
3859      __ srai(t0, t0, Interpreter::logStackElementSize);
3860      __ sd(t0, monitor_block_bot);         // set new monitor block bottom
3861 
3862      __ j(entry);
3863      // 2. move expression stack contents
3864      __ bind(loop);
3865      __ ld(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
3866                                                    // word from old location
3867      __ sd(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
3868      __ addi(c_rarg3, c_rarg3, wordSize);          // advance to next word
3869      __ bind(entry);
3870      __ bne(c_rarg3, c_rarg1, loop);    // check if bottom reached.if not at bottom
3871                                         // then copy next word
3872    }
3873 
3874    // call run-time routine
3875    // c_rarg1: points to monitor entry
3876    __ bind(allocated);
3877 
3878    // Increment bcp to point to the next bytecode, so exception
3879    // handling for async. exceptions work correctly.
3880    // The object has already been popped from the stack, so the
3881    // expression stack looks correct.
3882    __ addi(xbcp, xbcp, 1);
3883 
3884    // store object
3885    __ sd(x10, Address(c_rarg1, BasicObjectLock::obj_offset()));
3886    __ lock_object(c_rarg1);
3887 
3888    // check to make sure this monitor doesn't cause stack overflow after locking
3889    __ save_bcp();  // in case of exception
3890    __ generate_stack_overflow_check(0);
3891 
3892    // The bcp has already been incremented. Just need to dispatch to
3893    // next instruction.
3894    __ dispatch_next(vtos);
3895 }
3896 
3897 void TemplateTable::monitorexit() {
3898   transition(atos, vtos);
3899 
3900   // check for null object
3901   __ null_check(x10);
3902 
3903   const Address monitor_block_top(
3904         fp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3905   const Address monitor_block_bot(
3906         fp, frame::interpreter_frame_initial_sp_offset * wordSize);
3907   const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
3908 
3909   Label found;
3910 
3911   // find matching slot
3912   {
3913     Label entry, loop;
3914     __ ld(c_rarg1, monitor_block_top); // derelativize pointer
3915     __ shadd(c_rarg1, c_rarg1, fp, c_rarg1, LogBytesPerWord);
3916     // Now c_rarg1 points to current entry, starting with top-most entry
3917 
3918     __ la(c_rarg2, monitor_block_bot); // points to word before bottom
3919                                         // of monitor block
3920     __ j(entry);
3921 
3922     __ bind(loop);
3923     // check if current entry is for same object
3924     __ ld(t0, Address(c_rarg1, BasicObjectLock::obj_offset()));
3925     // if same object then stop searching
3926     __ beq(x10, t0, found);
3927     // otherwise advance to next entry
3928     __ add(c_rarg1, c_rarg1, entry_size);
3929     __ bind(entry);
3930     // check if bottom reached
3931     // if not at bottom then check this entry
3932     __ bne(c_rarg1, c_rarg2, loop);
3933   }
3934 
3935   // error handling. Unlocking was not block-structured
3936   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3937              InterpreterRuntime::throw_illegal_monitor_state_exception));
3938   __ should_not_reach_here();
3939 
3940   // call run-time routine
3941   __ bind(found);
3942   __ push_ptr(x10); // make sure object is on stack (contract with oopMaps)
3943   __ unlock_object(c_rarg1);
3944   __ pop_ptr(x10); // discard object
3945 }
3946 
3947 // Wide instructions
3948 void TemplateTable::wide() {
3949   __ load_unsigned_byte(x9, at_bcp(1));
3950   __ mv(t0, (address)Interpreter::_wentry_point);
3951   __ shadd(t0, x9, t0, t1, 3);
3952   __ ld(t1, Address(t0));
3953   __ jr(t1);
3954 }
3955 
3956 // Multi arrays
3957 void TemplateTable::multianewarray() {
3958   transition(vtos, atos);
3959   __ load_unsigned_byte(x10, at_bcp(3)); // get number of dimensions
3960   // last dim is on top of stack; we want address of first one:
3961   // first_addr = last_addr + (ndims - 1) * wordSize
3962   __ shadd(c_rarg1, x10, esp, c_rarg1, 3);
3963   __ subi(c_rarg1, c_rarg1, wordSize);
3964   call_VM(x10,
3965           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
3966           c_rarg1);
3967   __ load_unsigned_byte(x11, at_bcp(3));
3968   __ shadd(esp, x11, esp, t0, 3);
3969 }