1 /*
   2  * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.inline.hpp"
  29 #include "gc/shared/barrierSetAssembler.hpp"
  30 #include "gc/shared/collectedHeap.hpp"
  31 #include "gc/shared/tlab_globals.hpp"
  32 #include "interpreter/interp_masm.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "interpreter/interpreterRuntime.hpp"
  35 #include "interpreter/templateTable.hpp"
  36 #include "memory/universe.hpp"
  37 #include "oops/method.hpp"
  38 #include "oops/methodData.hpp"
  39 #include "oops/objArrayKlass.hpp"
  40 #include "oops/oop.inline.hpp"
  41 #include "prims/jvmtiExport.hpp"
  42 #include "prims/methodHandles.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 #include "runtime/synchronizer.hpp"
  47 #include "utilities/powerOfTwo.hpp"
  48 
  49 #define __ _masm->
  50 
  51 // Address computation: local variables
  52 
  53 static inline Address iaddress(int n) {
  54   return Address(xlocals, Interpreter::local_offset_in_bytes(n));
  55 }
  56 
  57 static inline Address laddress(int n) {
  58   return iaddress(n + 1);
  59 }
  60 
  61 static inline Address faddress(int n) {
  62   return iaddress(n);
  63 }
  64 
  65 static inline Address daddress(int n) {
  66   return laddress(n);
  67 }
  68 
  69 static inline Address aaddress(int n) {
  70   return iaddress(n);
  71 }
  72 
  73 static inline Address iaddress(Register r,  Register temp, InterpreterMacroAssembler* _masm) {
  74   assert_cond(_masm != NULL);
  75   _masm->shadd(temp, r, xlocals, temp, 3);
  76   return Address(temp, 0);
  77 }
  78 
  79 static inline Address laddress(Register r, Register temp,
  80                                InterpreterMacroAssembler* _masm) {
  81   assert_cond(_masm != NULL);
  82   _masm->shadd(temp, r, xlocals, temp, 3);
  83   return Address(temp, Interpreter::local_offset_in_bytes(1));;
  84 }
  85 
  86 static inline Address faddress(Register r, Register temp, InterpreterMacroAssembler* _masm) {
  87   return iaddress(r, temp, _masm);
  88 }
  89 
  90 static inline Address daddress(Register r, Register temp,
  91                                InterpreterMacroAssembler* _masm) {
  92   return laddress(r, temp, _masm);
  93 }
  94 
  95 static inline Address aaddress(Register r, Register temp, InterpreterMacroAssembler* _masm) {
  96   return iaddress(r, temp, _masm);
  97 }
  98 
  99 static inline Address at_rsp() {
 100   return Address(esp, 0);
 101 }
 102 
 103 // At top of Java expression stack which may be different than esp().  It
 104 // isn't for category 1 objects.
 105 static inline Address at_tos   () {
 106   return Address(esp,  Interpreter::expr_offset_in_bytes(0));
 107 }
 108 
 109 static inline Address at_tos_p1() {
 110   return Address(esp,  Interpreter::expr_offset_in_bytes(1));
 111 }
 112 
 113 static inline Address at_tos_p2() {
 114   return Address(esp,  Interpreter::expr_offset_in_bytes(2));
 115 }
 116 
 117 static inline Address at_tos_p3() {
 118   return Address(esp,  Interpreter::expr_offset_in_bytes(3));
 119 }
 120 
 121 static inline Address at_tos_p4() {
 122   return Address(esp,  Interpreter::expr_offset_in_bytes(4));
 123 }
 124 
 125 static inline Address at_tos_p5() {
 126   return Address(esp,  Interpreter::expr_offset_in_bytes(5));
 127 }
 128 
 129 // Miscelaneous helper routines
 130 // Store an oop (or NULL) at the Address described by obj.
 131 // If val == noreg this means store a NULL
 132 static void do_oop_store(InterpreterMacroAssembler* _masm,
 133                          Address dst,
 134                          Register val,
 135                          DecoratorSet decorators) {
 136   assert(val == noreg || val == x10, "parameter is just for looks");
 137   assert_cond(_masm != NULL);
 138   __ store_heap_oop(dst, val, x29, x11, decorators);
 139 }
 140 
 141 static void do_oop_load(InterpreterMacroAssembler* _masm,
 142                         Address src,
 143                         Register dst,
 144                         DecoratorSet decorators) {
 145   assert_cond(_masm != NULL);
 146   __ load_heap_oop(dst, src, x7, x11, decorators);
 147 }
 148 
 149 Address TemplateTable::at_bcp(int offset) {
 150   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 151   return Address(xbcp, offset);
 152 }
 153 
 154 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 155                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 156                                    int byte_no)
 157 {
 158   if (!RewriteBytecodes)  { return; }
 159   Label L_patch_done;
 160 
 161   switch (bc) {
 162     case Bytecodes::_fast_aputfield:  // fall through
 163     case Bytecodes::_fast_bputfield:  // fall through
 164     case Bytecodes::_fast_zputfield:  // fall through
 165     case Bytecodes::_fast_cputfield:  // fall through
 166     case Bytecodes::_fast_dputfield:  // fall through
 167     case Bytecodes::_fast_fputfield:  // fall through
 168     case Bytecodes::_fast_iputfield:  // fall through
 169     case Bytecodes::_fast_lputfield:  // fall through
 170     case Bytecodes::_fast_sputfield: {
 171       // We skip bytecode quickening for putfield instructions when
 172       // the put_code written to the constant pool cache is zero.
 173       // This is required so that every execution of this instruction
 174       // calls out to InterpreterRuntime::resolve_get_put to do
 175       // additional, required work.
 176       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 177       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 178       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 179       __ mv(bc_reg, bc);
 180       __ beqz(temp_reg, L_patch_done);
 181       break;
 182     }
 183     default:
 184       assert(byte_no == -1, "sanity");
 185       // the pair bytecodes have already done the load.
 186       if (load_bc_into_bc_reg) {
 187         __ mv(bc_reg, bc);
 188       }
 189   }
 190 
 191   if (JvmtiExport::can_post_breakpoint()) {
 192     Label L_fast_patch;
 193     // if a breakpoint is present we can't rewrite the stream directly
 194     __ load_unsigned_byte(temp_reg, at_bcp(0));
 195     __ addi(temp_reg, temp_reg, -Bytecodes::_breakpoint); // temp_reg is temporary register.
 196     __ bnez(temp_reg, L_fast_patch);
 197     // Let breakpoint table handling rewrite to quicker bytecode
 198     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), xmethod, xbcp, bc_reg);
 199     __ j(L_patch_done);
 200     __ bind(L_fast_patch);
 201   }
 202 
 203 #ifdef ASSERT
 204   Label L_okay;
 205   __ load_unsigned_byte(temp_reg, at_bcp(0));
 206   __ beq(temp_reg, bc_reg, L_okay);
 207   __ addi(temp_reg, temp_reg, -(int) Bytecodes::java_code(bc));
 208   __ beqz(temp_reg, L_okay);
 209   __ stop("patching the wrong bytecode");
 210   __ bind(L_okay);
 211 #endif
 212 
 213   // patch bytecode
 214   __ sb(bc_reg, at_bcp(0));
 215   __ bind(L_patch_done);
 216 }
 217 
 218 // Individual instructions
 219 
 220 void TemplateTable::nop() {
 221   transition(vtos, vtos);
 222   // nothing to do
 223 }
 224 
 225 void TemplateTable::shouldnotreachhere() {
 226   transition(vtos, vtos);
 227   __ stop("should not reach here bytecode");
 228 }
 229 
 230 void TemplateTable::aconst_null()
 231 {
 232   transition(vtos, atos);
 233   __ mv(x10, zr);
 234 }
 235 
 236 void TemplateTable::iconst(int value)
 237 {
 238   transition(vtos, itos);
 239   __ li(x10, value);
 240 }
 241 
 242 void TemplateTable::lconst(int value)
 243 {
 244   transition(vtos, ltos);
 245   __ li(x10, value);
 246 }
 247 
 248 void TemplateTable::fconst(int value)
 249 {
 250   transition(vtos, ftos);
 251   static float fBuf[2] = {1.0, 2.0};
 252   __ mv(t0, (intptr_t)fBuf);
 253   switch (value) {
 254     case 0:
 255       __ fmv_w_x(f10, zr);
 256       break;
 257     case 1:
 258       __ flw(f10, t0, 0);
 259       break;
 260     case 2:
 261       __ flw(f10, t0, sizeof(float));
 262       break;
 263     default:
 264       ShouldNotReachHere();
 265   }
 266 }
 267 
 268 void TemplateTable::dconst(int value)
 269 {
 270   transition(vtos, dtos);
 271   static double dBuf[2] = {1.0, 2.0};
 272   __ mv(t0, (intptr_t)dBuf);
 273   switch (value) {
 274     case 0:
 275       __ fmv_d_x(f10, zr);
 276       break;
 277     case 1:
 278       __ fld(f10, t0, 0);
 279       break;
 280     case 2:
 281       __ fld(f10, t0, sizeof(double));
 282       break;
 283     default:
 284       ShouldNotReachHere();
 285   }
 286 }
 287 
 288 void TemplateTable::bipush()
 289 {
 290   transition(vtos, itos);
 291   __ load_signed_byte(x10, at_bcp(1));
 292 }
 293 
 294 void TemplateTable::sipush()
 295 {
 296   transition(vtos, itos);
 297   __ load_unsigned_short(x10, at_bcp(1));
 298   __ revb_w_w(x10, x10);
 299   __ sraiw(x10, x10, 16);
 300 }
 301 
 302 void TemplateTable::ldc(bool wide)
 303 {
 304   transition(vtos, vtos);
 305   Label call_ldc, notFloat, notClass, notInt, Done;
 306 
 307   if (wide) {
 308    __ get_unsigned_2_byte_index_at_bcp(x11, 1);
 309   } else {
 310    __ load_unsigned_byte(x11, at_bcp(1));
 311   }
 312   __ get_cpool_and_tags(x12, x10);
 313 
 314   const int base_offset = ConstantPool::header_size() * wordSize;
 315   const int tags_offset = Array<u1>::base_offset_in_bytes();
 316 
 317   // get type
 318   __ addi(x13, x11, tags_offset);
 319   __ add(x13, x10, x13);
 320   __ membar(MacroAssembler::AnyAny);
 321   __ lbu(x13, Address(x13, 0));
 322   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
 323 
 324   // unresolved class - get the resolved class
 325   __ mv(t1, (u1)JVM_CONSTANT_UnresolvedClass);
 326   __ beq(x13, t1, call_ldc);
 327 
 328   // unresolved class in error state - call into runtime to throw the error
 329   // from the first resolution attempt
 330   __ mv(t1, (u1)JVM_CONSTANT_UnresolvedClassInError);
 331   __ beq(x13, t1, call_ldc);
 332 
 333   // resolved class - need to call vm to get java mirror of the class
 334   __ mv(t1, (u1)JVM_CONSTANT_Class);
 335   __ bne(x13, t1, notClass);
 336 
 337   __ bind(call_ldc);
 338   __ mv(c_rarg1, wide);
 339   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 340   __ push_ptr(x10);
 341   __ verify_oop(x10);
 342   __ j(Done);
 343 
 344   __ bind(notClass);
 345   __ mv(t1, (u1)JVM_CONSTANT_Float);
 346   __ bne(x13, t1, notFloat);
 347 
 348   // ftos
 349   __ shadd(x11, x11, x12, x11, 3);
 350   __ flw(f10, Address(x11, base_offset));
 351   __ push_f(f10);
 352   __ j(Done);
 353 
 354   __ bind(notFloat);
 355 
 356   __ mv(t1, (u1)JVM_CONSTANT_Integer);
 357   __ bne(x13, t1, notInt);
 358 
 359   // itos
 360   __ shadd(x11, x11, x12, x11, 3);
 361   __ lw(x10, Address(x11, base_offset));
 362   __ push_i(x10);
 363   __ j(Done);
 364 
 365   __ bind(notInt);
 366   condy_helper(Done);
 367 
 368   __ bind(Done);
 369 }
 370 
 371 // Fast path for caching oop constants.
 372 void TemplateTable::fast_aldc(bool wide)
 373 {
 374   transition(vtos, atos);
 375 
 376   const Register result = x10;
 377   const Register tmp = x11;
 378   const Register rarg = x12;
 379 
 380   const int index_size = wide ? sizeof(u2) : sizeof(u1);
 381 
 382   Label resolved;
 383 
 384   // We are resolved if the resolved reference cache entry contains a
 385   // non-null object (String, MethodType, etc.)
 386   assert_different_registers(result, tmp);
 387   __ get_cache_index_at_bcp(tmp, 1, index_size);
 388   __ load_resolved_reference_at_index(result, tmp);
 389   __ bnez(result, resolved);
 390 
 391   const address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 392 
 393   // first time invocation - must resolve first
 394   __ mv(rarg, (int)bytecode());
 395   __ call_VM(result, entry, rarg);
 396 
 397   __ bind(resolved);
 398 
 399   { // Check for the null sentinel.
 400     // If we just called the VM, it already did the mapping for us,
 401     // but it's harmless to retry.
 402     Label notNull;
 403 
 404     // Stash null_sentinel address to get its value later
 405     int32_t offset = 0;
 406     __ movptr_with_offset(rarg, Universe::the_null_sentinel_addr(), offset);
 407     __ ld(tmp, Address(rarg, offset));
 408     __ resolve_oop_handle(tmp);
 409     __ bne(result, tmp, notNull);
 410     __ mv(result, zr);  // NULL object reference
 411     __ bind(notNull);
 412   }
 413 
 414   if (VerifyOops) {
 415     // Safe to call with 0 result
 416     __ verify_oop(result);
 417   }
 418 }
 419 
 420 void TemplateTable::ldc2_w()
 421 {
 422     transition(vtos, vtos);
 423     Label notDouble, notLong, Done;
 424     __ get_unsigned_2_byte_index_at_bcp(x10, 1);
 425 
 426     __ get_cpool_and_tags(x11, x12);
 427     const int base_offset = ConstantPool::header_size() * wordSize;
 428     const int tags_offset = Array<u1>::base_offset_in_bytes();
 429 
 430     // get type
 431     __ add(x12, x12, x10);
 432     __ load_unsigned_byte(x12, Address(x12, tags_offset));
 433     __ mv(t1, JVM_CONSTANT_Double);
 434     __ bne(x12, t1, notDouble);
 435 
 436     // dtos
 437     __ shadd(x12, x10, x11, x12, 3);
 438     __ fld(f10, Address(x12, base_offset));
 439     __ push_d(f10);
 440     __ j(Done);
 441 
 442     __ bind(notDouble);
 443     __ mv(t1, (int)JVM_CONSTANT_Long);
 444     __ bne(x12, t1, notLong);
 445 
 446     // ltos
 447     __ shadd(x10, x10, x11, x10, 3);
 448     __ ld(x10, Address(x10, base_offset));
 449     __ push_l(x10);
 450     __ j(Done);
 451 
 452     __ bind(notLong);
 453     condy_helper(Done);
 454     __ bind(Done);
 455 
 456 }
 457 
 458 void TemplateTable::condy_helper(Label& Done)
 459 {
 460   const Register obj = x10;
 461   const Register rarg = x11;
 462   const Register flags = x12;
 463   const Register off = x13;
 464 
 465   const address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 466 
 467   __ mv(rarg, (int) bytecode());
 468   __ call_VM(obj, entry, rarg);
 469 
 470   __ get_vm_result_2(flags, xthread);
 471 
 472   // VMr = obj = base address to find primitive value to push
 473   // VMr2 = flags = (tos, off) using format of CPCE::_flags
 474   __ mv(off, flags);
 475   __ mv(t0, ConstantPoolCacheEntry::field_index_mask);
 476   __ andrw(off, off, t0);
 477 
 478   __ add(off, obj, off);
 479   const Address field(off, 0); // base + R---->base + offset
 480 
 481   __ slli(flags, flags, XLEN - (ConstantPoolCacheEntry::tos_state_shift + ConstantPoolCacheEntry::tos_state_bits));
 482   __ srli(flags, flags, XLEN - ConstantPoolCacheEntry::tos_state_bits); // (1 << 5) - 4 --> 28~31==> flags:0~3
 483 
 484   switch (bytecode()) {
 485     case Bytecodes::_ldc:   // fall through
 486     case Bytecodes::_ldc_w: {
 487       // tos in (itos, ftos, stos, btos, ctos, ztos)
 488       Label notInt, notFloat, notShort, notByte, notChar, notBool;
 489       __ mv(t1, itos);
 490       __ bne(flags, t1, notInt);
 491       // itos
 492       __ lw(x10, field);
 493       __ push(itos);
 494       __ j(Done);
 495 
 496       __ bind(notInt);
 497       __ mv(t1, ftos);
 498       __ bne(flags, t1, notFloat);
 499       // ftos
 500       __ load_float(field);
 501       __ push(ftos);
 502       __ j(Done);
 503 
 504       __ bind(notFloat);
 505       __ mv(t1, stos);
 506       __ bne(flags, t1, notShort);
 507       // stos
 508       __ load_signed_short(x10, field);
 509       __ push(stos);
 510       __ j(Done);
 511 
 512       __ bind(notShort);
 513       __ mv(t1, btos);
 514       __ bne(flags, t1, notByte);
 515       // btos
 516       __ load_signed_byte(x10, field);
 517       __ push(btos);
 518       __ j(Done);
 519 
 520       __ bind(notByte);
 521       __ mv(t1, ctos);
 522       __ bne(flags, t1, notChar);
 523       // ctos
 524       __ load_unsigned_short(x10, field);
 525       __ push(ctos);
 526       __ j(Done);
 527 
 528       __ bind(notChar);
 529       __ mv(t1, ztos);
 530       __ bne(flags, t1, notBool);
 531       // ztos
 532       __ load_signed_byte(x10, field);
 533       __ push(ztos);
 534       __ j(Done);
 535 
 536       __ bind(notBool);
 537       break;
 538     }
 539 
 540     case Bytecodes::_ldc2_w: {
 541       Label notLong, notDouble;
 542       __ mv(t1, ltos);
 543       __ bne(flags, t1, notLong);
 544       // ltos
 545       __ ld(x10, field);
 546       __ push(ltos);
 547       __ j(Done);
 548 
 549       __ bind(notLong);
 550       __ mv(t1, dtos);
 551       __ bne(flags, t1, notDouble);
 552       // dtos
 553       __ load_double(field);
 554       __ push(dtos);
 555       __ j(Done);
 556 
 557       __ bind(notDouble);
 558       break;
 559     }
 560 
 561     default:
 562       ShouldNotReachHere();
 563   }
 564 
 565   __ stop("bad ldc/condy");
 566 }
 567 
 568 void TemplateTable::locals_index(Register reg, int offset)
 569 {
 570   __ lbu(reg, at_bcp(offset));
 571   __ neg(reg, reg);
 572 }
 573 
 574 void TemplateTable::iload() {
 575   iload_internal();
 576 }
 577 
 578 void TemplateTable::nofast_iload() {
 579   iload_internal(may_not_rewrite);
 580 }
 581 
 582 void TemplateTable::iload_internal(RewriteControl rc) {
 583   transition(vtos, itos);
 584   if (RewriteFrequentPairs && rc == may_rewrite) {
 585     Label rewrite, done;
 586     const Register bc = x14;
 587 
 588     // get next bytecode
 589     __ load_unsigned_byte(x11, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 590 
 591     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 592     // last two iloads in a pair.  Comparing against fast_iload means that
 593     // the next bytecode is neither an iload or a caload, and therefore
 594     // an iload pair.
 595     __ mv(t1, Bytecodes::_iload);
 596     __ beq(x11, t1, done);
 597 
 598     // if _fast_iload rewrite to _fast_iload2
 599     __ mv(t1, Bytecodes::_fast_iload);
 600     __ mv(bc, Bytecodes::_fast_iload2);
 601     __ beq(x11, t1, rewrite);
 602 
 603     // if _caload rewrite to _fast_icaload
 604     __ mv(t1, Bytecodes::_caload);
 605     __ mv(bc, Bytecodes::_fast_icaload);
 606     __ beq(x11, t1, rewrite);
 607 
 608     // else rewrite to _fast_iload
 609     __ mv(bc, Bytecodes::_fast_iload);
 610 
 611     // rewrite
 612     // bc: new bytecode
 613     __ bind(rewrite);
 614     patch_bytecode(Bytecodes::_iload, bc, x11, false);
 615     __ bind(done);
 616 
 617   }
 618 
 619   // do iload, get the local value into tos
 620   locals_index(x11);
 621   __ lw(x10, iaddress(x11, x10, _masm));
 622 }
 623 
 624 void TemplateTable::fast_iload2()
 625 {
 626   transition(vtos, itos);
 627   locals_index(x11);
 628   __ lw(x10, iaddress(x11, x10, _masm));
 629   __ push(itos);
 630   locals_index(x11, 3);
 631   __ lw(x10, iaddress(x11, x10, _masm));
 632 }
 633 
 634 void TemplateTable::fast_iload()
 635 {
 636   transition(vtos, itos);
 637   locals_index(x11);
 638   __ lw(x10, iaddress(x11, x10, _masm));
 639 }
 640 
 641 void TemplateTable::lload()
 642 {
 643   transition(vtos, ltos);
 644   __ lbu(x11, at_bcp(1));
 645   __ slli(x11, x11, LogBytesPerWord);
 646   __ sub(x11, xlocals, x11);
 647   __ ld(x10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 648 }
 649 
 650 void TemplateTable::fload()
 651 {
 652   transition(vtos, ftos);
 653   locals_index(x11);
 654   __ flw(f10, faddress(x11, t0, _masm));
 655 }
 656 
 657 void TemplateTable::dload()
 658 {
 659   transition(vtos, dtos);
 660   __ lbu(x11, at_bcp(1));
 661   __ slli(x11, x11, LogBytesPerWord);
 662   __ sub(x11, xlocals, x11);
 663   __ fld(f10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 664 }
 665 
 666 void TemplateTable::aload()
 667 {
 668   transition(vtos, atos);
 669   locals_index(x11);
 670   __ ld(x10, iaddress(x11, x10, _masm));
 671 
 672 }
 673 
 674 void TemplateTable::locals_index_wide(Register reg) {
 675   __ lhu(reg, at_bcp(2));
 676   __ revb_h_h_u(reg, reg); // reverse bytes in half-word and zero-extend
 677   __ neg(reg, reg);
 678 }
 679 
 680 void TemplateTable::wide_iload() {
 681   transition(vtos, itos);
 682   locals_index_wide(x11);
 683   __ lw(x10, iaddress(x11, t0, _masm));
 684 }
 685 
 686 void TemplateTable::wide_lload()
 687 {
 688   transition(vtos, ltos);
 689   __ lhu(x11, at_bcp(2));
 690   __ revb_h_h_u(x11, x11); // reverse bytes in half-word and zero-extend
 691   __ slli(x11, x11, LogBytesPerWord);
 692   __ sub(x11, xlocals, x11);
 693   __ ld(x10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 694 }
 695 
 696 void TemplateTable::wide_fload()
 697 {
 698   transition(vtos, ftos);
 699   locals_index_wide(x11);
 700   __ flw(f10, faddress(x11, t0, _masm));
 701 }
 702 
 703 void TemplateTable::wide_dload()
 704 {
 705   transition(vtos, dtos);
 706   __ lhu(x11, at_bcp(2));
 707   __ revb_h_h_u(x11, x11); // reverse bytes in half-word and zero-extend
 708   __ slli(x11, x11, LogBytesPerWord);
 709   __ sub(x11, xlocals, x11);
 710   __ fld(f10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 711 }
 712 
 713 void TemplateTable::wide_aload()
 714 {
 715   transition(vtos, atos);
 716   locals_index_wide(x11);
 717   __ ld(x10, aaddress(x11, t0, _masm));
 718 }
 719 
 720 void TemplateTable::index_check(Register array, Register index)
 721 {
 722   // destroys x11, t0
 723   // check array
 724   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
 725   // sign extend index for use by indexed load
 726   // check index
 727   const Register length = t0;
 728   __ lwu(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 729   if (index != x11) {
 730     assert(x11 != array, "different registers");
 731     __ mv(x11, index);
 732   }
 733   Label ok;
 734   __ addw(index, index, zr);
 735   __ bltu(index, length, ok);
 736   __ mv(x13, array);
 737   __ mv(t0, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 738   __ jr(t0);
 739   __ bind(ok);
 740 }
 741 
 742 void TemplateTable::iaload()
 743 {
 744   transition(itos, itos);
 745   __ mv(x11, x10);
 746   __ pop_ptr(x10);
 747   // x10: array
 748   // x11: index
 749   index_check(x10, x11); // leaves index in x11
 750   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
 751   __ shadd(x10, x11, x10, t0, 2);
 752   __ access_load_at(T_INT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 753   __ addw(x10, x10, zr); // signed extended
 754 }
 755 
 756 void TemplateTable::laload()
 757 {
 758   transition(itos, ltos);
 759   __ mv(x11, x10);
 760   __ pop_ptr(x10);
 761   // x10: array
 762   // x11: index
 763   index_check(x10, x11); // leaves index in x11
 764   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
 765   __ shadd(x10, x11, x10, t0, 3);
 766   __ access_load_at(T_LONG, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 767 }
 768 
 769 void TemplateTable::faload()
 770 {
 771   transition(itos, ftos);
 772   __ mv(x11, x10);
 773   __ pop_ptr(x10);
 774   // x10: array
 775   // x11: index
 776   index_check(x10, x11); // leaves index in x11
 777   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
 778   __ shadd(x10, x11, x10, t0, 2);
 779   __ access_load_at(T_FLOAT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 780 }
 781 
 782 void TemplateTable::daload()
 783 {
 784   transition(itos, dtos);
 785   __ mv(x11, x10);
 786   __ pop_ptr(x10);
 787   // x10: array
 788   // x11: index
 789   index_check(x10, x11); // leaves index in x11
 790   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
 791   __ shadd(x10, x11, x10, t0, 3);
 792   __ access_load_at(T_DOUBLE, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 793 }
 794 
 795 void TemplateTable::aaload()
 796 {
 797   transition(itos, atos);
 798   __ mv(x11, x10);
 799   __ pop_ptr(x10);
 800   // x10: array
 801   // x11: index
 802   index_check(x10, x11); // leaves index in x11
 803   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 804   __ shadd(x10, x11, x10, t0, LogBytesPerHeapOop);
 805   do_oop_load(_masm,
 806               Address(x10),
 807               x10,
 808               IS_ARRAY);
 809 }
 810 
 811 void TemplateTable::baload()
 812 {
 813   transition(itos, itos);
 814   __ mv(x11, x10);
 815   __ pop_ptr(x10);
 816   // x10: array
 817   // x11: index
 818   index_check(x10, x11); // leaves index in x11
 819   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
 820   __ shadd(x10, x11, x10, t0, 0);
 821   __ access_load_at(T_BYTE, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 822 }
 823 
 824 void TemplateTable::caload()
 825 {
 826  transition(itos, itos);
 827   __ mv(x11, x10);
 828   __ pop_ptr(x10);
 829   // x10: array
 830   // x11: index
 831   index_check(x10, x11); // leaves index in x11
 832   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
 833   __ shadd(x10, x11, x10, t0, 1);
 834   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 835 }
 836 
 837 // iload followed by caload frequent pair
 838 void TemplateTable::fast_icaload()
 839 {
 840   transition(vtos, itos);
 841   // load index out of locals
 842   locals_index(x12);
 843   __ lw(x11, iaddress(x12, x11, _masm));
 844   __ pop_ptr(x10);
 845 
 846   // x10: array
 847   // x11: index
 848   index_check(x10, x11); // leaves index in x11, kills t0
 849   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1); // addi, max imm is 2^11
 850   __ shadd(x10, x11, x10, t0, 1);
 851   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 852 }
 853 
 854 void TemplateTable::saload()
 855 {
 856   transition(itos, itos);
 857   __ mv(x11, x10);
 858   __ pop_ptr(x10);
 859   // x10: array
 860   // x11: index
 861   index_check(x10, x11); // leaves index in x11, kills t0
 862   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_SHORT) >> 1);
 863   __ shadd(x10, x11, x10, t0, 1);
 864   __ access_load_at(T_SHORT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 865 }
 866 
 867 void TemplateTable::iload(int n)
 868 {
 869   transition(vtos, itos);
 870   __ lw(x10, iaddress(n));
 871 }
 872 
 873 void TemplateTable::lload(int n)
 874 {
 875   transition(vtos, ltos);
 876   __ ld(x10, laddress(n));
 877 }
 878 
 879 void TemplateTable::fload(int n)
 880 {
 881   transition(vtos, ftos);
 882   __ flw(f10, faddress(n));
 883 }
 884 
 885 void TemplateTable::dload(int n)
 886 {
 887   transition(vtos, dtos);
 888   __ fld(f10, daddress(n));
 889 }
 890 
 891 void TemplateTable::aload(int n)
 892 {
 893   transition(vtos, atos);
 894   __ ld(x10, iaddress(n));
 895 }
 896 
 897 void TemplateTable::aload_0() {
 898   aload_0_internal();
 899 }
 900 
 901 void TemplateTable::nofast_aload_0() {
 902   aload_0_internal(may_not_rewrite);
 903 }
 904 
 905 void TemplateTable::aload_0_internal(RewriteControl rc) {
 906   // According to bytecode histograms, the pairs:
 907   //
 908   // _aload_0, _fast_igetfield
 909   // _aload_0, _fast_agetfield
 910   // _aload_0, _fast_fgetfield
 911   //
 912   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 913   // _aload_0 bytecode checks if the next bytecode is either
 914   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 915   // rewrites the current bytecode into a pair bytecode; otherwise it
 916   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 917   // the pair check anymore.
 918   //
 919   // Note: If the next bytecode is _getfield, the rewrite must be
 920   //       delayed, otherwise we may miss an opportunity for a pair.
 921   //
 922   // Also rewrite frequent pairs
 923   //   aload_0, aload_1
 924   //   aload_0, iload_1
 925   // These bytecodes with a small amount of code are most profitable
 926   // to rewrite
 927   if (RewriteFrequentPairs && rc == may_rewrite) {
 928     Label rewrite, done;
 929     const Register bc = x14;
 930 
 931     // get next bytecode
 932     __ load_unsigned_byte(x11, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 933 
 934     // if _getfield then wait with rewrite
 935     __ mv(t1, Bytecodes::Bytecodes::_getfield);
 936     __ beq(x11, t1, done);
 937 
 938     // if _igetfield then rewrite to _fast_iaccess_0
 939     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 940     __ mv(t1, Bytecodes::_fast_igetfield);
 941     __ mv(bc, Bytecodes::_fast_iaccess_0);
 942     __ beq(x11, t1, rewrite);
 943 
 944     // if _agetfield then rewrite to _fast_aaccess_0
 945     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 946     __ mv(t1, Bytecodes::_fast_agetfield);
 947     __ mv(bc, Bytecodes::_fast_aaccess_0);
 948     __ beq(x11, t1, rewrite);
 949 
 950     // if _fgetfield then rewrite to _fast_faccess_0
 951     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 952     __ mv(t1, Bytecodes::_fast_fgetfield);
 953     __ mv(bc, Bytecodes::_fast_faccess_0);
 954     __ beq(x11, t1, rewrite);
 955 
 956     // else rewrite to _fast_aload0
 957     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 958     __ mv(bc, Bytecodes::Bytecodes::_fast_aload_0);
 959 
 960     // rewrite
 961     // bc: new bytecode
 962     __ bind(rewrite);
 963     patch_bytecode(Bytecodes::_aload_0, bc, x11, false);
 964 
 965     __ bind(done);
 966   }
 967 
 968   // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
 969   aload(0);
 970 }
 971 
 972 void TemplateTable::istore()
 973 {
 974   transition(itos, vtos);
 975   locals_index(x11);
 976   __ sw(x10, iaddress(x11, t0, _masm));
 977 }
 978 
 979 void TemplateTable::lstore()
 980 {
 981   transition(ltos, vtos);
 982   locals_index(x11);
 983   __ sd(x10, laddress(x11, t0, _masm));
 984 }
 985 
 986 void TemplateTable::fstore() {
 987   transition(ftos, vtos);
 988   locals_index(x11);
 989   __ fsw(f10, iaddress(x11, t0, _masm));
 990 }
 991 
 992 void TemplateTable::dstore() {
 993   transition(dtos, vtos);
 994   locals_index(x11);
 995   __ fsd(f10, daddress(x11, t0, _masm));
 996 }
 997 
 998 void TemplateTable::astore()
 999 {
1000   transition(vtos, vtos);
1001   __ pop_ptr(x10);
1002   locals_index(x11);
1003   __ sd(x10, aaddress(x11, t0, _masm));
1004 }
1005 
1006 void TemplateTable::wide_istore() {
1007   transition(vtos, vtos);
1008   __ pop_i();
1009   locals_index_wide(x11);
1010   __ sw(x10, iaddress(x11, t0, _masm));
1011 }
1012 
1013 void TemplateTable::wide_lstore() {
1014   transition(vtos, vtos);
1015   __ pop_l();
1016   locals_index_wide(x11);
1017   __ sd(x10, laddress(x11, t0, _masm));
1018 }
1019 
1020 void TemplateTable::wide_fstore() {
1021   transition(vtos, vtos);
1022   __ pop_f();
1023   locals_index_wide(x11);
1024   __ fsw(f10, faddress(x11, t0, _masm));
1025 }
1026 
1027 void TemplateTable::wide_dstore() {
1028   transition(vtos, vtos);
1029   __ pop_d();
1030   locals_index_wide(x11);
1031   __ fsd(f10, daddress(x11, t0, _masm));
1032 }
1033 
1034 void TemplateTable::wide_astore() {
1035   transition(vtos, vtos);
1036   __ pop_ptr(x10);
1037   locals_index_wide(x11);
1038   __ sd(x10, aaddress(x11, t0, _masm));
1039 }
1040 
1041 void TemplateTable::iastore() {
1042   transition(itos, vtos);
1043   __ pop_i(x11);
1044   __ pop_ptr(x13);
1045   // x10: value
1046   // x11: index
1047   // x13: array
1048   index_check(x13, x11); // prefer index in x11
1049   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
1050   __ shadd(t0, x11, x13, t0, 2);
1051   __ access_store_at(T_INT, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg);
1052 }
1053 
1054 void TemplateTable::lastore() {
1055   transition(ltos, vtos);
1056   __ pop_i(x11);
1057   __ pop_ptr(x13);
1058   // x10: value
1059   // x11: index
1060   // x13: array
1061   index_check(x13, x11); // prefer index in x11
1062   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
1063   __ shadd(t0, x11, x13, t0, 3);
1064   __ access_store_at(T_LONG, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg);
1065 }
1066 
1067 void TemplateTable::fastore() {
1068   transition(ftos, vtos);
1069   __ pop_i(x11);
1070   __ pop_ptr(x13);
1071   // f10: value
1072   // x11:  index
1073   // x13:  array
1074   index_check(x13, x11); // prefer index in x11
1075   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
1076   __ shadd(t0, x11, x13, t0, 2);
1077   __ access_store_at(T_FLOAT, IN_HEAP | IS_ARRAY, Address(t0, 0), noreg /* ftos */, noreg, noreg);
1078 }
1079 
1080 void TemplateTable::dastore() {
1081   transition(dtos, vtos);
1082   __ pop_i(x11);
1083   __ pop_ptr(x13);
1084   // f10: value
1085   // x11:  index
1086   // x13:  array
1087   index_check(x13, x11); // prefer index in x11
1088   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
1089   __ shadd(t0, x11, x13, t0, 3);
1090   __ access_store_at(T_DOUBLE, IN_HEAP | IS_ARRAY, Address(t0, 0), noreg /* dtos */, noreg, noreg);
1091 }
1092 
1093 void TemplateTable::aastore() {
1094   Label is_null, ok_is_subtype, done;
1095   transition(vtos, vtos);
1096   // stack: ..., array, index, value
1097   __ ld(x10, at_tos());    // value
1098   __ ld(x12, at_tos_p1()); // index
1099   __ ld(x13, at_tos_p2()); // array
1100 
1101   index_check(x13, x12);     // kills x11
1102   __ add(x14, x12, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
1103   __ shadd(x14, x14, x13, x14, LogBytesPerHeapOop);
1104 
1105   Address element_address(x14, 0);
1106 
1107   // do array store check - check for NULL value first
1108   __ beqz(x10, is_null);
1109 
1110   // Move subklass into x11
1111   __ load_klass(x11, x10);
1112   // Move superklass into x10
1113   __ load_klass(x10, x13);
1114   __ ld(x10, Address(x10,
1115                      ObjArrayKlass::element_klass_offset()));
1116   // Compress array + index * oopSize + 12 into a single register.  Frees x12.
1117 
1118   // Generate subtype check.  Blows x12, x15
1119   // Superklass in x10.  Subklass in x11.
1120   __ gen_subtype_check(x11, ok_is_subtype); //todo
1121 
1122   // Come here on failure
1123   // object is at TOS
1124   __ j(Interpreter::_throw_ArrayStoreException_entry);
1125 
1126   // Come here on success
1127   __ bind(ok_is_subtype);
1128 
1129   // Get the value we will store
1130   __ ld(x10, at_tos());
1131   // Now store using the appropriate barrier
1132   do_oop_store(_masm, element_address, x10, IS_ARRAY);
1133   __ j(done);
1134 
1135   // Have a NULL in x10, x13=array, x12=index.  Store NULL at ary[idx]
1136   __ bind(is_null);
1137   __ profile_null_seen(x12);
1138 
1139   // Store a NULL
1140   do_oop_store(_masm, element_address, noreg, IS_ARRAY);
1141 
1142   // Pop stack arguments
1143   __ bind(done);
1144   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1145 
1146 }
1147 
1148 void TemplateTable::bastore()
1149 {
1150   transition(itos, vtos);
1151   __ pop_i(x11);
1152   __ pop_ptr(x13);
1153   // x10: value
1154   // x11: index
1155   // x13: array
1156   index_check(x13, x11); // prefer index in x11
1157 
1158   // Need to check whether array is boolean or byte
1159   // since both types share the bastore bytecode.
1160   __ load_klass(x12, x13);
1161   __ lwu(x12, Address(x12, Klass::layout_helper_offset()));
1162   Label L_skip;
1163   __ andi(t0, x12, Klass::layout_helper_boolean_diffbit());
1164   __ beqz(t0, L_skip);
1165   __ andi(x10, x10, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1166   __ bind(L_skip);
1167 
1168   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
1169 
1170   __ add(x11, x13, x11);
1171   __ access_store_at(T_BYTE, IN_HEAP | IS_ARRAY, Address(x11, 0), x10, noreg, noreg);
1172 }
1173 
1174 void TemplateTable::castore()
1175 {
1176   transition(itos, vtos);
1177   __ pop_i(x11);
1178   __ pop_ptr(x13);
1179   // x10: value
1180   // x11: index
1181   // x13: array
1182   index_check(x13, x11); // prefer index in x11
1183   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
1184   __ shadd(t0, x11, x13, t0, 1);
1185   __ access_store_at(T_CHAR, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg);
1186 }
1187 
1188 void TemplateTable::sastore()
1189 {
1190   castore();
1191 }
1192 
1193 void TemplateTable::istore(int n)
1194 {
1195   transition(itos, vtos);
1196   __ sd(x10, iaddress(n));
1197 }
1198 
1199 void TemplateTable::lstore(int n)
1200 {
1201   transition(ltos, vtos);
1202   __ sd(x10, laddress(n));
1203 }
1204 
1205 void TemplateTable::fstore(int n)
1206 {
1207   transition(ftos, vtos);
1208   __ fsw(f10, faddress(n));
1209 }
1210 
1211 void TemplateTable::dstore(int n)
1212 {
1213   transition(dtos, vtos);
1214   __ fsd(f10, daddress(n));
1215 }
1216 
1217 void TemplateTable::astore(int n)
1218 {
1219   transition(vtos, vtos);
1220   __ pop_ptr(x10);
1221   __ sd(x10, iaddress(n));
1222 }
1223 
1224 void TemplateTable::pop()
1225 {
1226   transition(vtos, vtos);
1227   __ addi(esp, esp, Interpreter::stackElementSize);
1228 }
1229 
1230 void TemplateTable::pop2()
1231 {
1232   transition(vtos, vtos);
1233   __ addi(esp, esp, 2 * Interpreter::stackElementSize);
1234 }
1235 
1236 void TemplateTable::dup()
1237 {
1238   transition(vtos, vtos);
1239   __ ld(x10, Address(esp, 0));
1240   __ push_reg(x10);
1241   // stack: ..., a, a
1242 }
1243 
1244 void TemplateTable::dup_x1()
1245 {
1246   transition(vtos, vtos);
1247   // stack: ..., a, b
1248   __ ld(x10, at_tos());  // load b
1249   __ ld(x12, at_tos_p1());  // load a
1250   __ sd(x10, at_tos_p1());  // store b
1251   __ sd(x12, at_tos());  // store a
1252   __ push_reg(x10);                  // push b
1253   // stack: ..., b, a, b
1254 }
1255 
1256 void TemplateTable::dup_x2()
1257 {
1258   transition(vtos, vtos);
1259   // stack: ..., a, b, c
1260   __ ld(x10, at_tos());  // load c
1261   __ ld(x12, at_tos_p2());  // load a
1262   __ sd(x10, at_tos_p2());  // store c in a
1263   __ push_reg(x10);      // push c
1264   // stack: ..., c, b, c, c
1265   __ ld(x10, at_tos_p2());  // load b
1266   __ sd(x12, at_tos_p2());  // store a in b
1267   // stack: ..., c, a, c, c
1268   __ sd(x10, at_tos_p1());  // store b in c
1269   // stack: ..., c, a, b, c
1270 }
1271 
1272 void TemplateTable::dup2()
1273 {
1274   transition(vtos, vtos);
1275   // stack: ..., a, b
1276   __ ld(x10, at_tos_p1());  // load a
1277   __ push_reg(x10);                  // push a
1278   __ ld(x10, at_tos_p1());  // load b
1279   __ push_reg(x10);                  // push b
1280   // stack: ..., a, b, a, b
1281 }
1282 
1283 void TemplateTable::dup2_x1()
1284 {
1285   transition(vtos, vtos);
1286   // stack: ..., a, b, c
1287   __ ld(x12, at_tos());     // load c
1288   __ ld(x10, at_tos_p1());  // load b
1289   __ push_reg(x10);             // push b
1290   __ push_reg(x12);             // push c
1291   // stack: ..., a, b, c, b, c
1292   __ sd(x12, at_tos_p3());  // store c in b
1293   // stack: ..., a, c, c, b, c
1294   __ ld(x12, at_tos_p4());  // load a
1295   __ sd(x12, at_tos_p2());  // store a in 2nd c
1296   // stack: ..., a, c, a, b, c
1297   __ sd(x10, at_tos_p4());  // store b in a
1298   // stack: ..., b, c, a, b, c
1299 }
1300 
1301 void TemplateTable::dup2_x2()
1302 {
1303   transition(vtos, vtos);
1304   // stack: ..., a, b, c, d
1305   __ ld(x12, at_tos());     // load d
1306   __ ld(x10, at_tos_p1());  // load c
1307   __ push_reg(x10);             // push c
1308   __ push_reg(x12);             // push d
1309   // stack: ..., a, b, c, d, c, d
1310   __ ld(x10, at_tos_p4());  // load b
1311   __ sd(x10, at_tos_p2());  // store b in d
1312   __ sd(x12, at_tos_p4());  // store d in b
1313   // stack: ..., a, d, c, b, c, d
1314   __ ld(x12, at_tos_p5());  // load a
1315   __ ld(x10, at_tos_p3());  // load c
1316   __ sd(x12, at_tos_p3());  // store a in c
1317   __ sd(x10, at_tos_p5());  // store c in a
1318   // stack: ..., c, d, a, b, c, d
1319 }
1320 
1321 void TemplateTable::swap()
1322 {
1323   transition(vtos, vtos);
1324   // stack: ..., a, b
1325   __ ld(x12, at_tos_p1());  // load a
1326   __ ld(x10, at_tos());     // load b
1327   __ sd(x12, at_tos());     // store a in b
1328   __ sd(x10, at_tos_p1());  // store b in a
1329   // stack: ..., b, a
1330 }
1331 
1332 void TemplateTable::iop2(Operation op)
1333 {
1334   transition(itos, itos);
1335   // x10 <== x11 op x10
1336   __ pop_i(x11);
1337   switch (op) {
1338     case add  : __ addw(x10, x11, x10);  break;
1339     case sub  : __ subw(x10, x11, x10);  break;
1340     case mul  : __ mulw(x10, x11, x10);  break;
1341     case _and : __ andrw(x10, x11, x10); break;
1342     case _or  : __ orrw(x10, x11, x10);  break;
1343     case _xor : __ xorrw(x10, x11, x10); break;
1344     case shl  : __ sllw(x10, x11, x10);  break;
1345     case shr  : __ sraw(x10, x11, x10);  break;
1346     case ushr : __ srlw(x10, x11, x10);  break;
1347     default   : ShouldNotReachHere();
1348   }
1349 }
1350 
1351 void TemplateTable::lop2(Operation op)
1352 {
1353   transition(ltos, ltos);
1354   // x10 <== x11 op x10
1355   __ pop_l(x11);
1356   switch (op) {
1357     case add  : __ add(x10, x11, x10);  break;
1358     case sub  : __ sub(x10, x11, x10);  break;
1359     case mul  : __ mul(x10, x11, x10);  break;
1360     case _and : __ andr(x10, x11, x10); break;
1361     case _or  : __ orr(x10, x11, x10);  break;
1362     case _xor : __ xorr(x10, x11, x10); break;
1363     default   : ShouldNotReachHere();
1364   }
1365 }
1366 
1367 void TemplateTable::idiv()
1368 {
1369   transition(itos, itos);
1370   // explicitly check for div0
1371   Label no_div0;
1372   __ bnez(x10, no_div0);
1373   __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
1374   __ jr(t0);
1375   __ bind(no_div0);
1376   __ pop_i(x11);
1377   // x10 <== x11 idiv x10
1378   __ corrected_idivl(x10, x11, x10, /* want_remainder */ false);
1379 }
1380 
1381 void TemplateTable::irem()
1382 {
1383   transition(itos, itos);
1384   // explicitly check for div0
1385   Label no_div0;
1386   __ bnez(x10, no_div0);
1387   __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
1388   __ jr(t0);
1389   __ bind(no_div0);
1390   __ pop_i(x11);
1391   // x10 <== x11 irem x10
1392   __ corrected_idivl(x10, x11, x10, /* want_remainder */ true);
1393 }
1394 
1395 void TemplateTable::lmul()
1396 {
1397   transition(ltos, ltos);
1398   __ pop_l(x11);
1399   __ mul(x10, x10, x11);
1400 }
1401 
1402 void TemplateTable::ldiv()
1403 {
1404   transition(ltos, ltos);
1405   // explicitly check for div0
1406   Label no_div0;
1407   __ bnez(x10, no_div0);
1408   __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
1409   __ jr(t0);
1410   __ bind(no_div0);
1411   __ pop_l(x11);
1412   // x10 <== x11 ldiv x10
1413   __ corrected_idivq(x10, x11, x10, /* want_remainder */ false);
1414 }
1415 
1416 void TemplateTable::lrem()
1417 {
1418   transition(ltos, ltos);
1419   // explicitly check for div0
1420   Label no_div0;
1421   __ bnez(x10, no_div0);
1422   __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
1423   __ jr(t0);
1424   __ bind(no_div0);
1425   __ pop_l(x11);
1426   // x10 <== x11 lrem x10
1427   __ corrected_idivq(x10, x11, x10, /* want_remainder */ true);
1428 }
1429 
1430 void TemplateTable::lshl()
1431 {
1432   transition(itos, ltos);
1433   // shift count is in x10
1434   __ pop_l(x11);
1435   __ sll(x10, x11, x10);
1436 }
1437 
1438 void TemplateTable::lshr()
1439 {
1440   transition(itos, ltos);
1441   // shift count is in x10
1442   __ pop_l(x11);
1443   __ sra(x10, x11, x10);
1444 }
1445 
1446 void TemplateTable::lushr()
1447 {
1448   transition(itos, ltos);
1449   // shift count is in x10
1450   __ pop_l(x11);
1451   __ srl(x10, x11, x10);
1452 }
1453 
1454 void TemplateTable::fop2(Operation op)
1455 {
1456   transition(ftos, ftos);
1457   switch (op) {
1458     case add:
1459       __ pop_f(f11);
1460       __ fadd_s(f10, f11, f10);
1461       break;
1462     case sub:
1463       __ pop_f(f11);
1464       __ fsub_s(f10, f11, f10);
1465       break;
1466     case mul:
1467       __ pop_f(f11);
1468       __ fmul_s(f10, f11, f10);
1469       break;
1470     case div:
1471       __ pop_f(f11);
1472       __ fdiv_s(f10, f11, f10);
1473       break;
1474     case rem:
1475       __ fmv_s(f11, f10);
1476       __ pop_f(f10);
1477       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem));
1478       break;
1479     default:
1480       ShouldNotReachHere();
1481   }
1482 }
1483 
1484 void TemplateTable::dop2(Operation op)
1485 {
1486   transition(dtos, dtos);
1487   switch (op) {
1488     case add:
1489       __ pop_d(f11);
1490       __ fadd_d(f10, f11, f10);
1491       break;
1492     case sub:
1493       __ pop_d(f11);
1494       __ fsub_d(f10, f11, f10);
1495       break;
1496     case mul:
1497       __ pop_d(f11);
1498       __ fmul_d(f10, f11, f10);
1499       break;
1500     case div:
1501       __ pop_d(f11);
1502       __ fdiv_d(f10, f11, f10);
1503       break;
1504     case rem:
1505       __ fmv_d(f11, f10);
1506       __ pop_d(f10);
1507       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem));
1508       break;
1509     default:
1510       ShouldNotReachHere();
1511   }
1512 }
1513 
1514 void TemplateTable::ineg()
1515 {
1516   transition(itos, itos);
1517   __ negw(x10, x10);
1518 }
1519 
1520 void TemplateTable::lneg()
1521 {
1522   transition(ltos, ltos);
1523   __ neg(x10, x10);
1524 }
1525 
1526 void TemplateTable::fneg()
1527 {
1528   transition(ftos, ftos);
1529   __ fneg_s(f10, f10);
1530 }
1531 
1532 void TemplateTable::dneg()
1533 {
1534   transition(dtos, dtos);
1535   __ fneg_d(f10, f10);
1536 }
1537 
1538 void TemplateTable::iinc()
1539 {
1540   transition(vtos, vtos);
1541   __ load_signed_byte(x11, at_bcp(2)); // get constant
1542   locals_index(x12);
1543   __ ld(x10, iaddress(x12, x10, _masm));
1544   __ addw(x10, x10, x11);
1545   __ sd(x10, iaddress(x12, t0, _masm));
1546 }
1547 
1548 void TemplateTable::wide_iinc()
1549 {
1550   transition(vtos, vtos);
1551   __ lwu(x11, at_bcp(2)); // get constant and index
1552   __ revb_h_w_u(x11, x11); // reverse bytes in half-word (32bit) and zero-extend
1553   __ zero_extend(x12, x11, 16);
1554   __ neg(x12, x12);
1555   __ slli(x11, x11, 32);
1556   __ srai(x11, x11, 48);
1557   __ ld(x10, iaddress(x12, t0, _masm));
1558   __ addw(x10, x10, x11);
1559   __ sd(x10, iaddress(x12, t0, _masm));
1560 }
1561 
1562 void TemplateTable::convert()
1563 {
1564   // Checking
1565 #ifdef ASSERT
1566   {
1567     TosState tos_in  = ilgl;
1568     TosState tos_out = ilgl;
1569     switch (bytecode()) {
1570       case Bytecodes::_i2l: // fall through
1571       case Bytecodes::_i2f: // fall through
1572       case Bytecodes::_i2d: // fall through
1573       case Bytecodes::_i2b: // fall through
1574       case Bytecodes::_i2c: // fall through
1575       case Bytecodes::_i2s: tos_in = itos; break;
1576       case Bytecodes::_l2i: // fall through
1577       case Bytecodes::_l2f: // fall through
1578       case Bytecodes::_l2d: tos_in = ltos; break;
1579       case Bytecodes::_f2i: // fall through
1580       case Bytecodes::_f2l: // fall through
1581       case Bytecodes::_f2d: tos_in = ftos; break;
1582       case Bytecodes::_d2i: // fall through
1583       case Bytecodes::_d2l: // fall through
1584       case Bytecodes::_d2f: tos_in = dtos; break;
1585       default             : ShouldNotReachHere();
1586     }
1587     switch (bytecode()) {
1588       case Bytecodes::_l2i: // fall through
1589       case Bytecodes::_f2i: // fall through
1590       case Bytecodes::_d2i: // fall through
1591       case Bytecodes::_i2b: // fall through
1592       case Bytecodes::_i2c: // fall through
1593       case Bytecodes::_i2s: tos_out = itos; break;
1594       case Bytecodes::_i2l: // fall through
1595       case Bytecodes::_f2l: // fall through
1596       case Bytecodes::_d2l: tos_out = ltos; break;
1597       case Bytecodes::_i2f: // fall through
1598       case Bytecodes::_l2f: // fall through
1599       case Bytecodes::_d2f: tos_out = ftos; break;
1600       case Bytecodes::_i2d: // fall through
1601       case Bytecodes::_l2d: // fall through
1602       case Bytecodes::_f2d: tos_out = dtos; break;
1603       default             : ShouldNotReachHere();
1604     }
1605     transition(tos_in, tos_out);
1606   }
1607 #endif // ASSERT
1608 
1609   // Conversion
1610   switch (bytecode()) {
1611     case Bytecodes::_i2l:
1612       __ sign_extend(x10, x10, 32);
1613       break;
1614     case Bytecodes::_i2f:
1615       __ fcvt_s_w(f10, x10);
1616       break;
1617     case Bytecodes::_i2d:
1618       __ fcvt_d_w(f10, x10);
1619       break;
1620     case Bytecodes::_i2b:
1621       __ sign_extend(x10, x10, 8);
1622       break;
1623     case Bytecodes::_i2c:
1624       __ zero_extend(x10, x10, 16);
1625       break;
1626     case Bytecodes::_i2s:
1627       __ sign_extend(x10, x10, 16);
1628       break;
1629     case Bytecodes::_l2i:
1630       __ addw(x10, x10, zr);
1631       break;
1632     case Bytecodes::_l2f:
1633       __ fcvt_s_l(f10, x10);
1634       break;
1635     case Bytecodes::_l2d:
1636       __ fcvt_d_l(f10, x10);
1637       break;
1638     case Bytecodes::_f2i:
1639       __ fcvt_w_s_safe(x10, f10);
1640       break;
1641     case Bytecodes::_f2l:
1642       __ fcvt_l_s_safe(x10, f10);
1643       break;
1644     case Bytecodes::_f2d:
1645       __ fcvt_d_s(f10, f10);
1646       break;
1647     case Bytecodes::_d2i:
1648       __ fcvt_w_d_safe(x10, f10);
1649       break;
1650     case Bytecodes::_d2l:
1651       __ fcvt_l_d_safe(x10, f10);
1652       break;
1653     case Bytecodes::_d2f:
1654       __ fcvt_s_d(f10, f10);
1655       break;
1656     default:
1657       ShouldNotReachHere();
1658   }
1659 }
1660 
1661 void TemplateTable::lcmp()
1662 {
1663   transition(ltos, itos);
1664   __ pop_l(x11);
1665   __ cmp_l2i(t0, x11, x10);
1666   __ mv(x10, t0);
1667 }
1668 
1669 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1670 {
1671   // For instruction feq, flt and fle, the result is 0 if either operand is NaN
1672   if (is_float) {
1673     __ pop_f(f11);
1674     // if unordered_result < 0:
1675     //   we want -1 for unordered or less than, 0 for equal and 1 for
1676     //   greater than.
1677     // else:
1678     //   we want -1 for less than, 0 for equal and 1 for unordered or
1679     //   greater than.
1680     // f11 primary, f10 secondary
1681     __ float_compare(x10, f11, f10, unordered_result);
1682   } else {
1683     __ pop_d(f11);
1684     // if unordered_result < 0:
1685     //   we want -1 for unordered or less than, 0 for equal and 1 for
1686     //   greater than.
1687     // else:
1688     //   we want -1 for less than, 0 for equal and 1 for unordered or
1689     //   greater than.
1690     // f11 primary, f10 secondary
1691     __ double_compare(x10, f11, f10, unordered_result);
1692   }
1693 }
1694 
1695 void TemplateTable::branch(bool is_jsr, bool is_wide)
1696 {
1697   // We might be moving to a safepoint.  The thread which calls
1698   // Interpreter::notice_safepoints() will effectively flush its cache
1699   // when it makes a system call, but we need to do something to
1700   // ensure that we see the changed dispatch table.
1701   __ membar(MacroAssembler::LoadLoad);
1702 
1703   __ profile_taken_branch(x10, x11);
1704   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1705                              InvocationCounter::counter_offset();
1706   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1707                               InvocationCounter::counter_offset();
1708 
1709   // load branch displacement
1710   if (!is_wide) {
1711     __ lhu(x12, at_bcp(1));
1712     __ revb_h_h(x12, x12); // reverse bytes in half-word and sign-extend
1713   } else {
1714     __ lwu(x12, at_bcp(1));
1715     __ revb_w_w(x12, x12); // reverse bytes in word and sign-extend
1716   }
1717 
1718   // Handle all the JSR stuff here, then exit.
1719   // It's much shorter and cleaner than intermingling with the non-JSR
1720   // normal-branch stuff occurring below.
1721 
1722   if (is_jsr) {
1723     // compute return address as bci
1724     __ ld(t1, Address(xmethod, Method::const_offset()));
1725     __ add(t1, t1,
1726            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1727     __ sub(x11, xbcp, t1);
1728     __ push_i(x11);
1729     // Adjust the bcp by the 16-bit displacement in x12
1730     __ add(xbcp, xbcp, x12);
1731     __ load_unsigned_byte(t0, Address(xbcp, 0));
1732     // load the next target bytecode into t0, it is the argument of dispatch_only
1733     __ dispatch_only(vtos, /*generate_poll*/true);
1734     return;
1735   }
1736 
1737   // Normal (non-jsr) branch handling
1738 
1739   // Adjust the bcp by the displacement in x12
1740   __ add(xbcp, xbcp, x12);
1741 
1742   assert(UseLoopCounter || !UseOnStackReplacement,
1743          "on-stack-replacement requires loop counters");
1744   Label backedge_counter_overflow;
1745   Label dispatch;
1746   if (UseLoopCounter) {
1747     // increment backedge counter for backward branches
1748     // x10: MDO
1749     // x11: MDO bumped taken-count
1750     // x12: target offset
1751     __ bgtz(x12, dispatch); // count only if backward branch
1752 
1753     // check if MethodCounters exists
1754     Label has_counters;
1755     __ ld(t0, Address(xmethod, Method::method_counters_offset()));
1756     __ bnez(t0, has_counters);
1757     __ push_reg(x10);
1758     __ push_reg(x11);
1759     __ push_reg(x12);
1760     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1761             InterpreterRuntime::build_method_counters), xmethod);
1762     __ pop_reg(x12);
1763     __ pop_reg(x11);
1764     __ pop_reg(x10);
1765     __ ld(t0, Address(xmethod, Method::method_counters_offset()));
1766     __ beqz(t0, dispatch); // No MethodCounters allocated, OutOfMemory
1767     __ bind(has_counters);
1768 
1769     Label no_mdo;
1770     int increment = InvocationCounter::count_increment;
1771     if (ProfileInterpreter) {
1772       // Are we profiling?
1773       __ ld(x11, Address(xmethod, in_bytes(Method::method_data_offset())));
1774       __ beqz(x11, no_mdo);
1775       // Increment the MDO backedge counter
1776       const Address mdo_backedge_counter(x11, in_bytes(MethodData::backedge_counter_offset()) +
1777                                          in_bytes(InvocationCounter::counter_offset()));
1778       const Address mask(x11, in_bytes(MethodData::backedge_mask_offset()));
1779       __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1780                                  x10, t0, false,
1781                                  UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1782       __ j(dispatch);
1783     }
1784     __ bind(no_mdo);
1785     // Increment backedge counter in MethodCounters*
1786     __ ld(t0, Address(xmethod, Method::method_counters_offset()));
1787     const Address mask(t0, in_bytes(MethodCounters::backedge_mask_offset()));
1788     __ increment_mask_and_jump(Address(t0, be_offset), increment, mask,
1789                                x10, t1, false,
1790                                UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1791     __ bind(dispatch);
1792   }
1793 
1794   // Pre-load the next target bytecode into t0
1795   __ load_unsigned_byte(t0, Address(xbcp, 0));
1796 
1797   // continue with the bytecode @ target
1798   // t0: target bytecode
1799   // xbcp: target bcp
1800   __ dispatch_only(vtos, /*generate_poll*/true);
1801 
1802   if (UseLoopCounter && UseOnStackReplacement) {
1803     // invocation counter overflow
1804     __ bind(backedge_counter_overflow);
1805     __ neg(x12, x12);
1806     __ add(x12, x12, xbcp);     // branch xbcp
1807     // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
1808     __ call_VM(noreg,
1809                CAST_FROM_FN_PTR(address,
1810                                 InterpreterRuntime::frequency_counter_overflow),
1811                x12);
1812     __ load_unsigned_byte(x11, Address(xbcp, 0));  // restore target bytecode
1813 
1814     // x10: osr nmethod (osr ok) or NULL (osr not possible)
1815     // w11: target bytecode
1816     // x12: temporary
1817     __ beqz(x10, dispatch);     // test result -- no osr if null
1818     // nmethod may have been invalidated (VM may block upon call_VM return)
1819     __ lbu(x12, Address(x10, nmethod::state_offset()));
1820     if (nmethod::in_use != 0) {
1821       __ sub(x12, x12, nmethod::in_use);
1822     }
1823     __ bnez(x12, dispatch);
1824 
1825     // We have the address of an on stack replacement routine in x10
1826     // We need to prepare to execute the OSR method. First we must
1827     // migrate the locals and monitors off of the stack.
1828 
1829     __ mv(x9, x10);                             // save the nmethod
1830 
1831     call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
1832 
1833     // x10 is OSR buffer, move it to expected parameter location
1834     __ mv(j_rarg0, x10);
1835 
1836     // remove activation
1837     // get sender esp
1838     __ ld(esp,
1839         Address(fp, frame::interpreter_frame_sender_sp_offset * wordSize));
1840     // remove frame anchor
1841     __ leave();
1842     // Ensure compiled code always sees stack at proper alignment
1843     __ andi(sp, esp, -16);
1844 
1845     // and begin the OSR nmethod
1846     __ ld(t0, Address(x9, nmethod::osr_entry_point_offset()));
1847     __ jr(t0);
1848   }
1849 }
1850 
1851 void TemplateTable::if_0cmp(Condition cc)
1852 {
1853   transition(itos, vtos);
1854   // assume branch is more often taken than not (loops use backward branches)
1855   Label not_taken;
1856 
1857   __ addw(x10, x10, zr);
1858   switch (cc) {
1859     case equal:
1860       __ bnez(x10, not_taken);
1861       break;
1862     case not_equal:
1863       __ beqz(x10, not_taken);
1864       break;
1865     case less:
1866       __ bgez(x10, not_taken);
1867       break;
1868     case less_equal:
1869       __ bgtz(x10, not_taken);
1870       break;
1871     case greater:
1872       __ blez(x10, not_taken);
1873       break;
1874     case greater_equal:
1875       __ bltz(x10, not_taken);
1876       break;
1877     default:
1878       break;
1879   }
1880 
1881   branch(false, false);
1882   __ bind(not_taken);
1883   __ profile_not_taken_branch(x10);
1884 }
1885 
1886 void TemplateTable::if_icmp(Condition cc)
1887 {
1888   transition(itos, vtos);
1889   // assume branch is more often taken than not (loops use backward branches)
1890   Label not_taken;
1891   __ pop_i(x11);
1892   __ addw(x10, x10, zr);
1893   switch (cc) {
1894     case equal:
1895       __ bne(x11, x10, not_taken);
1896       break;
1897     case not_equal:
1898       __ beq(x11, x10, not_taken);
1899       break;
1900     case less:
1901       __ bge(x11, x10, not_taken);
1902       break;
1903     case less_equal:
1904       __ bgt(x11, x10, not_taken);
1905       break;
1906     case greater:
1907       __ ble(x11, x10, not_taken);
1908       break;
1909     case greater_equal:
1910       __ blt(x11, x10, not_taken);
1911       break;
1912     default:
1913       break;
1914   }
1915 
1916   branch(false, false);
1917   __ bind(not_taken);
1918   __ profile_not_taken_branch(x10);
1919 }
1920 
1921 void TemplateTable::if_nullcmp(Condition cc)
1922 {
1923   transition(atos, vtos);
1924   // assume branch is more often taken than not (loops use backward branches)
1925   Label not_taken;
1926   if (cc == equal) {
1927     __ bnez(x10, not_taken);
1928   } else {
1929     __ beqz(x10, not_taken);
1930   }
1931   branch(false, false);
1932   __ bind(not_taken);
1933   __ profile_not_taken_branch(x10);
1934 }
1935 
1936 void TemplateTable::if_acmp(Condition cc)
1937 {
1938   transition(atos, vtos);
1939   // assume branch is more often taken than not (loops use backward branches)
1940   Label not_taken;
1941   __ pop_ptr(x11);
1942 
1943   if (cc == equal) {
1944     __ bne(x11, x10, not_taken);
1945   } else if (cc == not_equal) {
1946     __ beq(x11, x10, not_taken);
1947   }
1948   branch(false, false);
1949   __ bind(not_taken);
1950   __ profile_not_taken_branch(x10);
1951 }
1952 
1953 void TemplateTable::ret() {
1954   transition(vtos, vtos);
1955   // We might be moving to a safepoint.  The thread which calls
1956   // Interpreter::notice_safepoints() will effectively flush its cache
1957   // when it makes a system call, but we need to do something to
1958   // ensure that we see the changed dispatch table.
1959   __ membar(MacroAssembler::LoadLoad);
1960 
1961   locals_index(x11);
1962   __ ld(x11, aaddress(x11, t1, _masm)); // get return bci, compute return bcp
1963   __ profile_ret(x11, x12);
1964   __ ld(xbcp, Address(xmethod, Method::const_offset()));
1965   __ add(xbcp, xbcp, x11);
1966   __ addi(xbcp, xbcp, in_bytes(ConstMethod::codes_offset()));
1967   __ dispatch_next(vtos, 0, /*generate_poll*/true);
1968 }
1969 
1970 void TemplateTable::wide_ret() {
1971   transition(vtos, vtos);
1972   locals_index_wide(x11);
1973   __ ld(x11, aaddress(x11, t0, _masm)); // get return bci, compute return bcp
1974   __ profile_ret(x11, x12);
1975   __ ld(xbcp, Address(xmethod, Method::const_offset()));
1976   __ add(xbcp, xbcp, x11);
1977   __ add(xbcp, xbcp, in_bytes(ConstMethod::codes_offset()));
1978   __ dispatch_next(vtos, 0, /*generate_poll*/true);
1979 }
1980 
1981 void TemplateTable::tableswitch() {
1982   Label default_case, continue_execution;
1983   transition(itos, vtos);
1984   // align xbcp
1985   __ la(x11, at_bcp(BytesPerInt));
1986   __ andi(x11, x11, -BytesPerInt);
1987   // load lo & hi
1988   __ lwu(x12, Address(x11, BytesPerInt));
1989   __ lwu(x13, Address(x11, 2 * BytesPerInt));
1990   __ revb_w_w(x12, x12); // reverse bytes in word (32bit) and sign-extend
1991   __ revb_w_w(x13, x13); // reverse bytes in word (32bit) and sign-extend
1992   // check against lo & hi
1993   __ blt(x10, x12, default_case);
1994   __ bgt(x10, x13, default_case);
1995   // lookup dispatch offset
1996   __ subw(x10, x10, x12);
1997   __ shadd(x13, x10, x11, t0, 2);
1998   __ lwu(x13, Address(x13, 3 * BytesPerInt));
1999   __ profile_switch_case(x10, x11, x12);
2000   // continue execution
2001   __ bind(continue_execution);
2002   __ revb_w_w(x13, x13); // reverse bytes in word (32bit) and sign-extend
2003   __ add(xbcp, xbcp, x13);
2004   __ load_unsigned_byte(t0, Address(xbcp));
2005   __ dispatch_only(vtos, /*generate_poll*/true);
2006   // handle default
2007   __ bind(default_case);
2008   __ profile_switch_default(x10);
2009   __ lwu(x13, Address(x11, 0));
2010   __ j(continue_execution);
2011 }
2012 
2013 void TemplateTable::lookupswitch() {
2014   transition(itos, itos);
2015   __ stop("lookupswitch bytecode should have been rewritten");
2016 }
2017 
2018 void TemplateTable::fast_linearswitch() {
2019   transition(itos, vtos);
2020   Label loop_entry, loop, found, continue_execution;
2021   // bswap x10 so we can avoid bswapping the table entries
2022   __ revb_w_w(x10, x10); // reverse bytes in word (32bit) and sign-extend
2023   // align xbcp
2024   __ la(x9, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2025                                     // this instruction (change offsets
2026                                     // below)
2027   __ andi(x9, x9, -BytesPerInt);
2028   // set counter
2029   __ lwu(x11, Address(x9, BytesPerInt));
2030   __ revb_w(x11, x11);
2031   __ j(loop_entry);
2032   // table search
2033   __ bind(loop);
2034   __ shadd(t0, x11, x9, t0, 3);
2035   __ lw(t0, Address(t0, 2 * BytesPerInt));
2036   __ beq(x10, t0, found);
2037   __ bind(loop_entry);
2038   __ addi(x11, x11, -1);
2039   __ bgez(x11, loop);
2040   // default case
2041   __ profile_switch_default(x10);
2042   __ lwu(x13, Address(x9, 0));
2043   __ j(continue_execution);
2044   // entry found -> get offset
2045   __ bind(found);
2046   __ shadd(t0, x11, x9, t0, 3);
2047   __ lwu(x13, Address(t0, 3 * BytesPerInt));
2048   __ profile_switch_case(x11, x10, x9);
2049   // continue execution
2050   __ bind(continue_execution);
2051   __ revb_w_w(x13, x13); // reverse bytes in word (32bit) and sign-extend
2052   __ add(xbcp, xbcp, x13);
2053   __ lbu(t0, Address(xbcp, 0));
2054   __ dispatch_only(vtos, /*generate_poll*/true);
2055 }
2056 
2057 void TemplateTable::fast_binaryswitch() {
2058   transition(itos, vtos);
2059   // Implementation using the following core algorithm:
2060   //
2061   // int binary_search(int key, LookupswitchPair* array, int n)
2062   //   binary_search start:
2063   //   #Binary search according to "Methodik des Programmierens" by
2064   //   # Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2065   //   int i = 0;
2066   //   int j = n;
2067   //   while (i + 1 < j) do
2068   //     # invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2069   //     # with      Q: for all i: 0 <= i < n: key < a[i]
2070   //     # where a stands for the array and assuming that the (inexisting)
2071   //     # element a[n] is infinitely big.
2072   //     int h = (i + j) >> 1
2073   //     # i < h < j
2074   //     if (key < array[h].fast_match())
2075   //     then [j = h]
2076   //     else [i = h]
2077   //   end
2078   //   # R: a[i] <= key < a[i+1] or Q
2079   //   # (i.e., if key is within array, i is the correct index)
2080   //   return i
2081   // binary_search end
2082 
2083 
2084   // Register allocation
2085   const Register key   = x10; // already set (tosca)
2086   const Register array = x11;
2087   const Register i     = x12;
2088   const Register j     = x13;
2089   const Register h     = x14;
2090   const Register temp  = x15;
2091 
2092   // Find array start
2093   __ la(array, at_bcp(3 * BytesPerInt));  // btw: should be able to
2094                                           // get rid of this
2095                                           // instruction (change
2096                                           // offsets below)
2097   __ andi(array, array, -BytesPerInt);
2098 
2099   // Initialize i & j
2100   __ mv(i, zr);                            // i = 0
2101   __ lwu(j, Address(array, -BytesPerInt)); // j = length(array)
2102 
2103   // Convert j into native byteordering
2104   __ revb_w(j, j);
2105 
2106   // And start
2107   Label entry;
2108   __ j(entry);
2109 
2110   // binary search loop
2111   {
2112     Label loop;
2113     __ bind(loop);
2114     __ addw(h, i, j);                           // h = i + j
2115     __ srliw(h, h, 1);                          // h = (i + j) >> 1
2116     // if [key < array[h].fast_match()]
2117     // then [j = h]
2118     // else [i = h]
2119     // Convert array[h].match to native byte-ordering before compare
2120     __ shadd(temp, h, array, temp, 3);
2121     __ ld(temp, Address(temp, 0));
2122     __ revb_w_w(temp, temp); // reverse bytes in word (32bit) and sign-extend
2123 
2124     Label L_done, L_greater;
2125     __ bge(key, temp, L_greater);
2126     // if [key < array[h].fast_match()] then j = h
2127     __ mv(j, h);
2128     __ j(L_done);
2129     __ bind(L_greater);
2130     // if [key >= array[h].fast_match()] then i = h
2131     __ mv(i, h);
2132     __ bind(L_done);
2133 
2134     // while [i + 1 < j]
2135     __ bind(entry);
2136     __ addiw(h, i, 1);         // i + 1
2137     __ blt(h, j, loop);        // i + 1 < j
2138   }
2139 
2140   // end of binary search, result index is i (must check again!)
2141   Label default_case;
2142   // Convert array[i].match to native byte-ordering before compare
2143   __ shadd(temp, i, array, temp, 3);
2144   __ ld(temp, Address(temp, 0));
2145   __ revb_w_w(temp, temp); // reverse bytes in word (32bit) and sign-extend
2146   __ bne(key, temp, default_case);
2147 
2148   // entry found -> j = offset
2149   __ shadd(temp, i, array, temp, 3);
2150   __ lwu(j, Address(temp, BytesPerInt));
2151   __ profile_switch_case(i, key, array);
2152   __ revb_w_w(j, j); // reverse bytes in word (32bit) and sign-extend
2153 
2154   __ add(temp, xbcp, j);
2155   __ load_unsigned_byte(t0, Address(temp, 0));
2156 
2157   __ add(xbcp, xbcp, j);
2158   __ la(xbcp, Address(xbcp, 0));
2159   __ dispatch_only(vtos, /*generate_poll*/true);
2160 
2161   // default case -> j = default offset
2162   __ bind(default_case);
2163   __ profile_switch_default(i);
2164   __ lwu(j, Address(array, -2 * BytesPerInt));
2165   __ revb_w_w(j, j); // reverse bytes in word (32bit) and sign-extend
2166 
2167   __ add(temp, xbcp, j);
2168   __ load_unsigned_byte(t0, Address(temp, 0));
2169 
2170   __ add(xbcp, xbcp, j);
2171   __ la(xbcp, Address(xbcp, 0));
2172   __ dispatch_only(vtos, /*generate_poll*/true);
2173 }
2174 
2175 void TemplateTable::_return(TosState state)
2176 {
2177   transition(state, state);
2178   assert(_desc->calls_vm(),
2179          "inconsistent calls_vm information"); // call in remove_activation
2180 
2181   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2182     assert(state == vtos, "only valid state");
2183 
2184     __ ld(c_rarg1, aaddress(0));
2185     __ load_klass(x13, c_rarg1);
2186     __ lwu(x13, Address(x13, Klass::access_flags_offset()));
2187     Label skip_register_finalizer;
2188     __ andi(t0, x13, JVM_ACC_HAS_FINALIZER);
2189     __ beqz(t0, skip_register_finalizer);
2190 
2191     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2192 
2193     __ bind(skip_register_finalizer);
2194   }
2195 
2196   // Issue a StoreStore barrier after all stores but before return
2197   // from any constructor for any class with a final field. We don't
2198   // know if this is a finalizer, so we always do so.
2199   if (_desc->bytecode() == Bytecodes::_return) {
2200     __ membar(MacroAssembler::StoreStore);
2201   }
2202 
2203   // Narrow result if state is itos but result type is smaller.
2204   // Need to narrow in the return bytecode rather than in generate_return_entry
2205   // since compiled code callers expect the result to already be narrowed.
2206   if (state == itos) {
2207     __ narrow(x10);
2208   }
2209 
2210   __ remove_activation(state);
2211   __ ret();
2212 }
2213 
2214 
2215 // ----------------------------------------------------------------------------
2216 // Volatile variables demand their effects be made known to all CPU's
2217 // in order.  Store buffers on most chips allow reads & writes to
2218 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2219 // without some kind of memory barrier (i.e., it's not sufficient that
2220 // the interpreter does not reorder volatile references, the hardware
2221 // also must not reorder them).
2222 //
2223 // According to the new Java Memory Model (JMM):
2224 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2225 //     writes act as aquire & release, so:
2226 // (2) A read cannot let unrelated NON-volatile memory refs that
2227 //     happen after the read float up to before the read.  It's OK for
2228 //     non-volatile memory refs that happen before the volatile read to
2229 //     float down below it.
2230 // (3) Similar a volatile write cannot let unrelated NON-volatile
2231 //     memory refs that happen BEFORE the write float down to after the
2232 //     write.  It's OK for non-volatile memory refs that happen after the
2233 //     volatile write to float up before it.
2234 //
2235 // We only put in barriers around volatile refs (they are expensive),
2236 // not _between_ memory refs (that would require us to track the
2237 // flavor of the previous memory refs).  Requirements (2) and (3)
2238 // require some barriers before volatile stores and after volatile
2239 // loads.  These nearly cover requirement (1) but miss the
2240 // volatile-store-volatile-load case.  This final case is placed after
2241 // volatile-stores although it could just as well go before
2242 // volatile-loads.
2243 
2244 void TemplateTable::resolve_cache_and_index(int byte_no,
2245                                             Register Rcache,
2246                                             Register index,
2247                                             size_t index_size) {
2248   const Register temp = x9;
2249   assert_different_registers(Rcache, index, temp);
2250 
2251   Label resolved, clinit_barrier_slow;
2252 
2253   Bytecodes::Code code = bytecode();
2254   switch (code) {
2255     case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2256     case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2257     default: break;
2258   }
2259 
2260   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2261   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2262   __ mv(t0, (int) code);
2263   __ beq(temp, t0, resolved);
2264 
2265   // resolve first time through
2266   // Class initialization barrier slow path lands here as well.
2267   __ bind(clinit_barrier_slow);
2268 
2269   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2270   __ mv(temp, (int) code);
2271   __ call_VM(noreg, entry, temp);
2272 
2273   // Update registers with resolved info
2274   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2275   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2276   // so all clients ofthis method must be modified accordingly
2277   __ bind(resolved);
2278 
2279   // Class initialization barrier for static methods
2280   if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
2281     __ load_resolved_method_at_index(byte_no, temp, Rcache);
2282     __ load_method_holder(temp, temp);
2283     __ clinit_barrier(temp, t0, NULL, &clinit_barrier_slow);
2284   }
2285 }
2286 
2287 // The Rcache and index registers must be set before call
2288 // n.b unlike x86 cache already includes the index offset
2289 void TemplateTable::load_field_cp_cache_entry(Register obj,
2290                                               Register cache,
2291                                               Register index,
2292                                               Register off,
2293                                               Register flags,
2294                                               bool is_static = false) {
2295   assert_different_registers(cache, index, flags, off);
2296 
2297   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2298   // Field offset
2299   __ ld(off, Address(cache, in_bytes(cp_base_offset +
2300                                      ConstantPoolCacheEntry::f2_offset())));
2301   // Flags
2302   __ lwu(flags, Address(cache, in_bytes(cp_base_offset +
2303                                         ConstantPoolCacheEntry::flags_offset())));
2304 
2305   // klass overwrite register
2306   if (is_static) {
2307     __ ld(obj, Address(cache, in_bytes(cp_base_offset +
2308                                        ConstantPoolCacheEntry::f1_offset())));
2309     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2310     __ ld(obj, Address(obj, mirror_offset));
2311     __ resolve_oop_handle(obj);
2312   }
2313 }
2314 
2315 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2316                                                Register method,
2317                                                Register itable_index,
2318                                                Register flags,
2319                                                bool is_invokevirtual,
2320                                                bool is_invokevfinal, /*unused*/
2321                                                bool is_invokedynamic) {
2322   // setup registers
2323   const Register cache = t1;
2324   const Register index = x14;
2325   assert_different_registers(method, flags);
2326   assert_different_registers(method, cache, index);
2327   assert_different_registers(itable_index, flags);
2328   assert_different_registers(itable_index, cache, index);
2329   // determine constant pool cache field offsets
2330   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2331   const int method_offset = in_bytes(ConstantPoolCache::base_offset() +
2332                                      (is_invokevirtual ?
2333                                       ConstantPoolCacheEntry::f2_offset() :
2334                                       ConstantPoolCacheEntry::f1_offset()));
2335   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2336                                     ConstantPoolCacheEntry::flags_offset());
2337   // access constant pool cache fields
2338   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2339                                     ConstantPoolCacheEntry::f2_offset());
2340 
2341   const size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2342   resolve_cache_and_index(byte_no, cache, index, index_size);
2343   __ ld(method, Address(cache, method_offset));
2344 
2345   if (itable_index != noreg) {
2346     __ ld(itable_index, Address(cache, index_offset));
2347   }
2348   __ lwu(flags, Address(cache, flags_offset));
2349 }
2350 
2351 // The registers cache and index expected to be set before call.
2352 // Correct values of the cache and index registers are preserved.
2353 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2354                                             bool is_static, bool has_tos) {
2355   // do the JVMTI work here to avoid disturbing the register state below
2356   // We use c_rarg registers here beacause we want to use the register used in
2357   // the call to the VM
2358   if (JvmtiExport::can_post_field_access()) {
2359     // Check to see if a field access watch has been set before we
2360     // take the time to call into the VM.
2361     Label L1;
2362     assert_different_registers(cache, index, x10);
2363     int32_t offset = 0;
2364     __ la_patchable(t0, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()), offset);
2365     __ lwu(x10, Address(t0, offset));
2366 
2367     __ beqz(x10, L1);
2368 
2369     __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2370     __ la(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2371 
2372     if (is_static) {
2373       __ mv(c_rarg1, zr); // NULL object reference
2374     } else {
2375       __ ld(c_rarg1, at_tos()); // get object pointer without popping it
2376       __ verify_oop(c_rarg1);
2377     }
2378     // c_rarg1: object pointer or NULL
2379     // c_rarg2: cache entry pointer
2380     // c_rarg3: jvalue object on the stack
2381     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2382                                        InterpreterRuntime::post_field_access),
2383                                        c_rarg1, c_rarg2, c_rarg3);
2384     __ get_cache_and_index_at_bcp(cache, index, 1);
2385     __ bind(L1);
2386   }
2387 }
2388 
2389 void TemplateTable::pop_and_check_object(Register r)
2390 {
2391   __ pop_ptr(r);
2392   __ null_check(r);  // for field access must check obj.
2393   __ verify_oop(r);
2394 }
2395 
2396 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc)
2397 {
2398   const Register cache     = x12;
2399   const Register index     = x13;
2400   const Register obj       = x14;
2401   const Register off       = x9;
2402   const Register flags     = x10;
2403   const Register raw_flags = x16;
2404   const Register bc        = x14; // uses same reg as obj, so don't mix them
2405 
2406   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2407   jvmti_post_field_access(cache, index, is_static, false);
2408   load_field_cp_cache_entry(obj, cache, index, off, raw_flags, is_static);
2409 
2410   if (!is_static) {
2411     // obj is on the stack
2412     pop_and_check_object(obj);
2413   }
2414 
2415   __ add(off, obj, off);
2416   const Address field(off);
2417 
2418   Label Done, notByte, notBool, notInt, notShort, notChar,
2419               notLong, notFloat, notObj, notDouble;
2420 
2421   __ slli(flags, raw_flags, XLEN - (ConstantPoolCacheEntry::tos_state_shift +
2422                                     ConstantPoolCacheEntry::tos_state_bits));
2423   __ srli(flags, flags, XLEN - ConstantPoolCacheEntry::tos_state_bits);
2424 
2425   assert(btos == 0, "change code, btos != 0");
2426   __ bnez(flags, notByte);
2427 
2428   // Dont't rewrite getstatic, only getfield
2429   if (is_static) {
2430     rc = may_not_rewrite;
2431   }
2432 
2433   // btos
2434   __ access_load_at(T_BYTE, IN_HEAP, x10, field, noreg, noreg);
2435   __ push(btos);
2436   // Rewrite bytecode to be faster
2437   if (rc == may_rewrite) {
2438     patch_bytecode(Bytecodes::_fast_bgetfield, bc, x11);
2439   }
2440   __ j(Done);
2441 
2442   __ bind(notByte);
2443   __ sub(t0, flags, (u1)ztos);
2444   __ bnez(t0, notBool);
2445 
2446   // ztos (same code as btos)
2447   __ access_load_at(T_BOOLEAN, IN_HEAP, x10, field, noreg, noreg);
2448   __ push(ztos);
2449   // Rewirte bytecode to be faster
2450   if (rc == may_rewrite) {
2451     // uses btos rewriting, no truncating to t/f bit is needed for getfield
2452     patch_bytecode(Bytecodes::_fast_bgetfield, bc, x11);
2453   }
2454   __ j(Done);
2455 
2456   __ bind(notBool);
2457   __ sub(t0, flags, (u1)atos);
2458   __ bnez(t0, notObj);
2459   // atos
2460   do_oop_load(_masm, field, x10, IN_HEAP);
2461   __ push(atos);
2462   if (rc == may_rewrite) {
2463     patch_bytecode(Bytecodes::_fast_agetfield, bc, x11);
2464   }
2465   __ j(Done);
2466 
2467   __ bind(notObj);
2468   __ sub(t0, flags, (u1)itos);
2469   __ bnez(t0, notInt);
2470   // itos
2471   __ access_load_at(T_INT, IN_HEAP, x10, field, noreg, noreg);
2472   __ addw(x10, x10, zr); // signed extended
2473   __ push(itos);
2474   // Rewrite bytecode to be faster
2475   if (rc == may_rewrite) {
2476     patch_bytecode(Bytecodes::_fast_igetfield, bc, x11);
2477   }
2478   __ j(Done);
2479 
2480   __ bind(notInt);
2481   __ sub(t0, flags, (u1)ctos);
2482   __ bnez(t0, notChar);
2483   // ctos
2484   __ access_load_at(T_CHAR, IN_HEAP, x10, field, noreg, noreg);
2485   __ push(ctos);
2486   // Rewrite bytecode to be faster
2487   if (rc == may_rewrite) {
2488     patch_bytecode(Bytecodes::_fast_cgetfield, bc, x11);
2489   }
2490   __ j(Done);
2491 
2492   __ bind(notChar);
2493   __ sub(t0, flags, (u1)stos);
2494   __ bnez(t0, notShort);
2495   // stos
2496   __ access_load_at(T_SHORT, IN_HEAP, x10, field, noreg, noreg);
2497   __ push(stos);
2498   // Rewrite bytecode to be faster
2499   if (rc == may_rewrite) {
2500     patch_bytecode(Bytecodes::_fast_sgetfield, bc, x11);
2501   }
2502   __ j(Done);
2503 
2504   __ bind(notShort);
2505   __ sub(t0, flags, (u1)ltos);
2506   __ bnez(t0, notLong);
2507   // ltos
2508   __ access_load_at(T_LONG, IN_HEAP, x10, field, noreg, noreg);
2509   __ push(ltos);
2510   // Rewrite bytecode to be faster
2511   if (rc == may_rewrite) {
2512     patch_bytecode(Bytecodes::_fast_lgetfield, bc, x11);
2513   }
2514   __ j(Done);
2515 
2516   __ bind(notLong);
2517   __ sub(t0, flags, (u1)ftos);
2518   __ bnez(t0, notFloat);
2519   // ftos
2520   __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2521   __ push(ftos);
2522   // Rewrite bytecode to be faster
2523   if (rc == may_rewrite) {
2524     patch_bytecode(Bytecodes::_fast_fgetfield, bc, x11);
2525   }
2526   __ j(Done);
2527 
2528   __ bind(notFloat);
2529 #ifdef ASSERT
2530   __ sub(t0, flags, (u1)dtos);
2531   __ bnez(t0, notDouble);
2532 #endif
2533   // dtos
2534   __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2535   __ push(dtos);
2536   // Rewrite bytecode to be faster
2537   if (rc == may_rewrite) {
2538     patch_bytecode(Bytecodes::_fast_dgetfield, bc, x11);
2539   }
2540 #ifdef ASSERT
2541   __ j(Done);
2542 
2543   __ bind(notDouble);
2544   __ stop("Bad state");
2545 #endif
2546 
2547   __ bind(Done);
2548 
2549   Label notVolatile;
2550   __ andi(t0, raw_flags, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
2551   __ beqz(t0, notVolatile);
2552   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2553   __ bind(notVolatile);
2554 }
2555 
2556 void TemplateTable::getfield(int byte_no)
2557 {
2558   getfield_or_static(byte_no, false);
2559 }
2560 
2561 void TemplateTable::nofast_getfield(int byte_no) {
2562   getfield_or_static(byte_no, false, may_not_rewrite);
2563 }
2564 
2565 void TemplateTable::getstatic(int byte_no)
2566 {
2567   getfield_or_static(byte_no, true);
2568 }
2569 
2570 // The registers cache and index expected to be set before call.
2571 // The function may destroy various registers, just not the cache and index registers.
2572 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2573   transition(vtos, vtos);
2574 
2575   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2576 
2577   if (JvmtiExport::can_post_field_modification()) {
2578     // Check to see if a field modification watch has been set before
2579     // we take the time to call into the VM.
2580     Label L1;
2581     assert_different_registers(cache, index, x10);
2582     int32_t offset = 0;
2583     __ la_patchable(t0, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()), offset);
2584     __ lwu(x10, Address(t0, offset));
2585     __ beqz(x10, L1);
2586 
2587     __ get_cache_and_index_at_bcp(c_rarg2, t0, 1);
2588 
2589     if (is_static) {
2590       // Life is simple. Null out the object pointer.
2591       __ mv(c_rarg1, zr);
2592     } else {
2593       // Life is harder. The stack holds the value on top, followed by
2594       // the object. We don't know the size of the value, though; it
2595       // could be one or two words depending on its type. As a result,
2596       // we must find the type to determine where the object is.
2597       __ lwu(c_rarg3, Address(c_rarg2,
2598                               in_bytes(cp_base_offset +
2599                                        ConstantPoolCacheEntry::flags_offset())));
2600       __ srli(c_rarg3, c_rarg3, ConstantPoolCacheEntry::tos_state_shift);
2601       ConstantPoolCacheEntry::verify_tos_state_shift();
2602       Label nope2, done, ok;
2603       __ ld(c_rarg1, at_tos_p1());   // initially assume a one word jvalue
2604       __ sub(t0, c_rarg3, ltos);
2605       __ beqz(t0, ok);
2606       __ sub(t0, c_rarg3, dtos);
2607       __ bnez(t0, nope2);
2608       __ bind(ok);
2609       __ ld(c_rarg1, at_tos_p2());  // ltos (two word jvalue);
2610       __ bind(nope2);
2611     }
2612     // cache entry pointer
2613     __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2614     // object (tos)
2615     __ mv(c_rarg3, esp);
2616     // c_rarg1: object pointer set up above (NULL if static)
2617     // c_rarg2: cache entry pointer
2618     // c_rarg3: jvalue object on  the stack
2619     __ call_VM(noreg,
2620                CAST_FROM_FN_PTR(address,
2621                                 InterpreterRuntime::post_field_modification),
2622                                 c_rarg1, c_rarg2, c_rarg3);
2623     __ get_cache_and_index_at_bcp(cache, index, 1);
2624     __ bind(L1);
2625   }
2626 }
2627 
2628 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2629   transition(vtos, vtos);
2630 
2631   const Register cache = x12;
2632   const Register index = x13;
2633   const Register obj   = x12;
2634   const Register off   = x9;
2635   const Register flags = x10;
2636   const Register bc    = x14;
2637 
2638   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2639   jvmti_post_field_mod(cache, index, is_static);
2640   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2641 
2642   Label Done;
2643   __ mv(x15, flags);
2644 
2645   {
2646     Label notVolatile;
2647     __ andi(t0, x15, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
2648     __ beqz(t0, notVolatile);
2649     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2650     __ bind(notVolatile);
2651   }
2652 
2653   Label notByte, notBool, notInt, notShort, notChar,
2654         notLong, notFloat, notObj, notDouble;
2655 
2656   __ slli(flags, flags, XLEN - (ConstantPoolCacheEntry::tos_state_shift +
2657                                 ConstantPoolCacheEntry::tos_state_bits));
2658   __ srli(flags, flags, XLEN - ConstantPoolCacheEntry::tos_state_bits);
2659 
2660   assert(btos == 0, "change code, btos != 0");
2661   __ bnez(flags, notByte);
2662 
2663   // Don't rewrite putstatic, only putfield
2664   if (is_static) {
2665     rc = may_not_rewrite;
2666   }
2667 
2668   // btos
2669   {
2670     __ pop(btos);
2671     // field address
2672     if (!is_static) {
2673       pop_and_check_object(obj);
2674     }
2675     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2676     const Address field(off, 0); // off register as temparator register.
2677     __ access_store_at(T_BYTE, IN_HEAP, field, x10, noreg, noreg);
2678     if (rc == may_rewrite) {
2679       patch_bytecode(Bytecodes::_fast_bputfield, bc, x11, true, byte_no);
2680     }
2681     __ j(Done);
2682   }
2683 
2684   __ bind(notByte);
2685   __ sub(t0, flags, (u1)ztos);
2686   __ bnez(t0, notBool);
2687 
2688   // ztos
2689   {
2690     __ pop(ztos);
2691     // field address
2692     if (!is_static) {
2693       pop_and_check_object(obj);
2694     }
2695     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2696     const Address field(off, 0);
2697     __ access_store_at(T_BOOLEAN, IN_HEAP, field, x10, noreg, noreg);
2698     if (rc == may_rewrite) {
2699       patch_bytecode(Bytecodes::_fast_zputfield, bc, x11, true, byte_no);
2700     }
2701     __ j(Done);
2702   }
2703 
2704   __ bind(notBool);
2705   __ sub(t0, flags, (u1)atos);
2706   __ bnez(t0, notObj);
2707 
2708   // atos
2709   {
2710     __ pop(atos);
2711     // field address
2712     if (!is_static) {
2713       pop_and_check_object(obj);
2714     }
2715     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2716     const Address field(off, 0);
2717     // Store into the field
2718     do_oop_store(_masm, field, x10, IN_HEAP);
2719     if (rc == may_rewrite) {
2720       patch_bytecode(Bytecodes::_fast_aputfield, bc, x11, true, byte_no);
2721     }
2722     __ j(Done);
2723   }
2724 
2725   __ bind(notObj);
2726   __ sub(t0, flags, (u1)itos);
2727   __ bnez(t0, notInt);
2728 
2729   // itos
2730   {
2731     __ pop(itos);
2732     // field address
2733     if (!is_static) {
2734       pop_and_check_object(obj);
2735     }
2736     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2737     const Address field(off, 0);
2738     __ access_store_at(T_INT, IN_HEAP, field, x10, noreg, noreg);
2739     if (rc == may_rewrite) {
2740       patch_bytecode(Bytecodes::_fast_iputfield, bc, x11, true, byte_no);
2741     }
2742     __ j(Done);
2743   }
2744 
2745   __ bind(notInt);
2746   __ sub(t0, flags, (u1)ctos);
2747   __ bnez(t0, notChar);
2748 
2749   // ctos
2750   {
2751     __ pop(ctos);
2752     // field address
2753     if (!is_static) {
2754       pop_and_check_object(obj);
2755     }
2756     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2757     const Address field(off, 0);
2758     __ access_store_at(T_CHAR, IN_HEAP, field, x10, noreg, noreg);
2759     if (rc == may_rewrite) {
2760       patch_bytecode(Bytecodes::_fast_cputfield, bc, x11, true, byte_no);
2761     }
2762     __ j(Done);
2763   }
2764 
2765   __ bind(notChar);
2766   __ sub(t0, flags, (u1)stos);
2767   __ bnez(t0, notShort);
2768 
2769   // stos
2770   {
2771     __ pop(stos);
2772     // field address
2773     if (!is_static) {
2774       pop_and_check_object(obj);
2775     }
2776     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2777     const Address field(off, 0);
2778     __ access_store_at(T_SHORT, IN_HEAP, field, x10, noreg, noreg);
2779     if (rc == may_rewrite) {
2780       patch_bytecode(Bytecodes::_fast_sputfield, bc, x11, true, byte_no);
2781     }
2782     __ j(Done);
2783   }
2784 
2785   __ bind(notShort);
2786   __ sub(t0, flags, (u1)ltos);
2787   __ bnez(t0, notLong);
2788 
2789   // ltos
2790   {
2791     __ pop(ltos);
2792     // field address
2793     if (!is_static) {
2794       pop_and_check_object(obj);
2795     }
2796     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2797     const Address field(off, 0);
2798     __ access_store_at(T_LONG, IN_HEAP, field, x10, noreg, noreg);
2799     if (rc == may_rewrite) {
2800       patch_bytecode(Bytecodes::_fast_lputfield, bc, x11, true, byte_no);
2801     }
2802     __ j(Done);
2803   }
2804 
2805   __ bind(notLong);
2806   __ sub(t0, flags, (u1)ftos);
2807   __ bnez(t0, notFloat);
2808 
2809   // ftos
2810   {
2811     __ pop(ftos);
2812     // field address
2813     if (!is_static) {
2814       pop_and_check_object(obj);
2815     }
2816     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2817     const Address field(off, 0);
2818     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg);
2819     if (rc == may_rewrite) {
2820       patch_bytecode(Bytecodes::_fast_fputfield, bc, x11, true, byte_no);
2821     }
2822     __ j(Done);
2823   }
2824 
2825   __ bind(notFloat);
2826 #ifdef ASSERT
2827   __ sub(t0, flags, (u1)dtos);
2828   __ bnez(t0, notDouble);
2829 #endif
2830 
2831   // dtos
2832   {
2833     __ pop(dtos);
2834     // field address
2835     if (!is_static) {
2836       pop_and_check_object(obj);
2837     }
2838     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2839     const Address field(off, 0);
2840     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg);
2841     if (rc == may_rewrite) {
2842       patch_bytecode(Bytecodes::_fast_dputfield, bc, x11, true, byte_no);
2843     }
2844   }
2845 
2846 #ifdef ASSERT
2847   __ j(Done);
2848 
2849   __ bind(notDouble);
2850   __ stop("Bad state");
2851 #endif
2852 
2853   __ bind(Done);
2854 
2855   {
2856     Label notVolatile;
2857     __ andi(t0, x15, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
2858     __ beqz(t0, notVolatile);
2859     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
2860     __ bind(notVolatile);
2861   }
2862 }
2863 
2864 void TemplateTable::putfield(int byte_no)
2865 {
2866   putfield_or_static(byte_no, false);
2867 }
2868 
2869 void TemplateTable::nofast_putfield(int byte_no) {
2870   putfield_or_static(byte_no, false, may_not_rewrite);
2871 }
2872 
2873 void TemplateTable::putstatic(int byte_no) {
2874   putfield_or_static(byte_no, true);
2875 }
2876 
2877 void TemplateTable::jvmti_post_fast_field_mod()
2878 {
2879   if (JvmtiExport::can_post_field_modification()) {
2880     // Check to see if a field modification watch has been set before
2881     // we take the time to call into the VM.
2882     Label L2;
2883     int32_t offset = 0;
2884     __ la_patchable(t0, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()), offset);
2885     __ lwu(c_rarg3, Address(t0, offset));
2886     __ beqz(c_rarg3, L2);
2887     __ pop_ptr(x9);                  // copy the object pointer from tos
2888     __ verify_oop(x9);
2889     __ push_ptr(x9);                 // put the object pointer back on tos
2890     // Save tos values before call_VM() clobbers them. Since we have
2891     // to do it for every data type, we use the saved values as the
2892     // jvalue object.
2893     switch (bytecode()) {          // load values into the jvalue object
2894       case Bytecodes::_fast_aputfield: __ push_ptr(x10); break;
2895       case Bytecodes::_fast_bputfield: // fall through
2896       case Bytecodes::_fast_zputfield: // fall through
2897       case Bytecodes::_fast_sputfield: // fall through
2898       case Bytecodes::_fast_cputfield: // fall through
2899       case Bytecodes::_fast_iputfield: __ push_i(x10); break;
2900       case Bytecodes::_fast_dputfield: __ push_d(); break;
2901       case Bytecodes::_fast_fputfield: __ push_f(); break;
2902       case Bytecodes::_fast_lputfield: __ push_l(x10); break;
2903 
2904       default:
2905         ShouldNotReachHere();
2906     }
2907     __ mv(c_rarg3, esp);             // points to jvalue on the stack
2908     // access constant pool cache entry
2909     __ get_cache_entry_pointer_at_bcp(c_rarg2, x10, 1);
2910     __ verify_oop(x9);
2911     // x9: object pointer copied above
2912     // c_rarg2: cache entry pointer
2913     // c_rarg3: jvalue object on the stack
2914     __ call_VM(noreg,
2915                CAST_FROM_FN_PTR(address,
2916                                 InterpreterRuntime::post_field_modification),
2917                x9, c_rarg2, c_rarg3);
2918 
2919     switch (bytecode()) {             // restore tos values
2920       case Bytecodes::_fast_aputfield: __ pop_ptr(x10); break;
2921       case Bytecodes::_fast_bputfield: // fall through
2922       case Bytecodes::_fast_zputfield: // fall through
2923       case Bytecodes::_fast_sputfield: // fall through
2924       case Bytecodes::_fast_cputfield: // fall through
2925       case Bytecodes::_fast_iputfield: __ pop_i(x10); break;
2926       case Bytecodes::_fast_dputfield: __ pop_d(); break;
2927       case Bytecodes::_fast_fputfield: __ pop_f(); break;
2928       case Bytecodes::_fast_lputfield: __ pop_l(x10); break;
2929       default: break;
2930     }
2931     __ bind(L2);
2932   }
2933 }
2934 
2935 void TemplateTable::fast_storefield(TosState state)
2936 {
2937   transition(state, vtos);
2938 
2939   ByteSize base = ConstantPoolCache::base_offset();
2940 
2941   jvmti_post_fast_field_mod();
2942 
2943   // access constant pool cache
2944   __ get_cache_and_index_at_bcp(x12, x11, 1);
2945 
2946   // Must prevent reordering of the following cp cache loads with bytecode load
2947   __ membar(MacroAssembler::LoadLoad);
2948 
2949   // test for volatile with x13
2950   __ lwu(x13, Address(x12, in_bytes(base +
2951                                     ConstantPoolCacheEntry::flags_offset())));
2952 
2953   // replace index with field offset from cache entry
2954   __ ld(x11, Address(x12, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
2955 
2956   {
2957     Label notVolatile;
2958     __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
2959     __ beqz(t0, notVolatile);
2960     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2961     __ bind(notVolatile);
2962   }
2963 
2964   // Get object from stack
2965   pop_and_check_object(x12);
2966 
2967   // field address
2968   __ add(x11, x12, x11);
2969   const Address field(x11, 0);
2970 
2971   // access field
2972   switch (bytecode()) {
2973     case Bytecodes::_fast_aputfield:
2974       do_oop_store(_masm, field, x10, IN_HEAP);
2975       break;
2976     case Bytecodes::_fast_lputfield:
2977       __ access_store_at(T_LONG, IN_HEAP, field, x10, noreg, noreg);
2978       break;
2979     case Bytecodes::_fast_iputfield:
2980       __ access_store_at(T_INT, IN_HEAP, field, x10, noreg, noreg);
2981       break;
2982     case Bytecodes::_fast_zputfield:
2983       __ access_store_at(T_BOOLEAN, IN_HEAP, field, x10, noreg, noreg);
2984       break;
2985     case Bytecodes::_fast_bputfield:
2986       __ access_store_at(T_BYTE, IN_HEAP, field, x10, noreg, noreg);
2987       break;
2988     case Bytecodes::_fast_sputfield:
2989       __ access_store_at(T_SHORT, IN_HEAP, field, x10, noreg, noreg);
2990       break;
2991     case Bytecodes::_fast_cputfield:
2992       __ access_store_at(T_CHAR, IN_HEAP, field, x10, noreg, noreg);
2993       break;
2994     case Bytecodes::_fast_fputfield:
2995       __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg);
2996       break;
2997     case Bytecodes::_fast_dputfield:
2998       __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg);
2999       break;
3000     default:
3001       ShouldNotReachHere();
3002   }
3003 
3004   {
3005     Label notVolatile;
3006     __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
3007     __ beqz(t0, notVolatile);
3008     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
3009     __ bind(notVolatile);
3010   }
3011 }
3012 
3013 void TemplateTable::fast_accessfield(TosState state)
3014 {
3015   transition(atos, state);
3016   // Do the JVMTI work here to avoid disturbing the register state below
3017   if (JvmtiExport::can_post_field_access()) {
3018     // Check to see if a field access watch has been set before we
3019     // take the time to call into the VM.
3020     Label L1;
3021     int32_t offset = 0;
3022     __ la_patchable(t0, ExternalAddress((address)JvmtiExport::get_field_access_count_addr()), offset);
3023     __ lwu(x12, Address(t0, offset));
3024     __ beqz(x12, L1);
3025     // access constant pool cache entry
3026     __ get_cache_entry_pointer_at_bcp(c_rarg2, t1, 1);
3027     __ verify_oop(x10);
3028     __ push_ptr(x10);  // save object pointer before call_VM() clobbers it
3029     __ mv(c_rarg1, x10);
3030     // c_rarg1: object pointer copied above
3031     // c_rarg2: cache entry pointer
3032     __ call_VM(noreg,
3033                CAST_FROM_FN_PTR(address,
3034                                 InterpreterRuntime::post_field_access),
3035                c_rarg1, c_rarg2);
3036     __ pop_ptr(x10); // restore object pointer
3037     __ bind(L1);
3038   }
3039 
3040   // access constant pool cache
3041   __ get_cache_and_index_at_bcp(x12, x11, 1);
3042 
3043   // Must prevent reordering of the following cp cache loads with bytecode load
3044   __ membar(MacroAssembler::LoadLoad);
3045 
3046   __ ld(x11, Address(x12, in_bytes(ConstantPoolCache::base_offset() +
3047                                    ConstantPoolCacheEntry::f2_offset())));
3048   __ lwu(x13, Address(x12, in_bytes(ConstantPoolCache::base_offset() +
3049                                     ConstantPoolCacheEntry::flags_offset())));
3050 
3051   // x10: object
3052   __ verify_oop(x10);
3053   __ null_check(x10);
3054   __ add(x11, x10, x11);
3055   const Address field(x11, 0);
3056 
3057   // access field
3058   switch (bytecode()) {
3059     case Bytecodes::_fast_agetfield:
3060       do_oop_load(_masm, field, x10, IN_HEAP);
3061       __ verify_oop(x10);
3062       break;
3063     case Bytecodes::_fast_lgetfield:
3064       __ access_load_at(T_LONG, IN_HEAP, x10, field, noreg, noreg);
3065       break;
3066     case Bytecodes::_fast_igetfield:
3067       __ access_load_at(T_INT, IN_HEAP, x10, field, noreg, noreg);
3068       __ addw(x10, x10, zr); // signed extended
3069       break;
3070     case Bytecodes::_fast_bgetfield:
3071       __ access_load_at(T_BYTE, IN_HEAP, x10, field, noreg, noreg);
3072       break;
3073     case Bytecodes::_fast_sgetfield:
3074       __ access_load_at(T_SHORT, IN_HEAP, x10, field, noreg, noreg);
3075       break;
3076     case Bytecodes::_fast_cgetfield:
3077       __ access_load_at(T_CHAR, IN_HEAP, x10, field, noreg, noreg);
3078       break;
3079     case Bytecodes::_fast_fgetfield:
3080       __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
3081       break;
3082     case Bytecodes::_fast_dgetfield:
3083       __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* dtos */, field, noreg, noreg);
3084       break;
3085     default:
3086       ShouldNotReachHere();
3087   }
3088   {
3089     Label notVolatile;
3090     __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
3091     __ beqz(t0, notVolatile);
3092     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3093     __ bind(notVolatile);
3094   }
3095 }
3096 
3097 void TemplateTable::fast_xaccess(TosState state)
3098 {
3099   transition(vtos, state);
3100 
3101   // get receiver
3102   __ ld(x10, aaddress(0));
3103   // access constant pool cache
3104   __ get_cache_and_index_at_bcp(x12, x13, 2);
3105   __ ld(x11, Address(x12, in_bytes(ConstantPoolCache::base_offset() +
3106                                    ConstantPoolCacheEntry::f2_offset())));
3107 
3108   // make sure exception is reported in correct bcp range (getfield is
3109   // next instruction)
3110   __ addi(xbcp, xbcp, 1);
3111   __ null_check(x10);
3112   switch (state) {
3113     case itos:
3114       __ add(x10, x10, x11);
3115       __ access_load_at(T_INT, IN_HEAP, x10, Address(x10, 0), noreg, noreg);
3116       __ addw(x10, x10, zr); // signed extended
3117       break;
3118     case atos:
3119       __ add(x10, x10, x11);
3120       do_oop_load(_masm, Address(x10, 0), x10, IN_HEAP);
3121       __ verify_oop(x10);
3122       break;
3123     case ftos:
3124       __ add(x10, x10, x11);
3125       __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, Address(x10), noreg, noreg);
3126       break;
3127     default:
3128       ShouldNotReachHere();
3129   }
3130 
3131   {
3132     Label notVolatile;
3133     __ lwu(x13, Address(x12, in_bytes(ConstantPoolCache::base_offset() +
3134                                       ConstantPoolCacheEntry::flags_offset())));
3135     __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
3136     __ beqz(t0, notVolatile);
3137     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3138     __ bind(notVolatile);
3139   }
3140 
3141   __ sub(xbcp, xbcp, 1);
3142 }
3143 
3144 //-----------------------------------------------------------------------------
3145 // Calls
3146 
3147 void TemplateTable::prepare_invoke(int byte_no,
3148                                    Register method, // linked method (or i-klass)
3149                                    Register index,  // itable index, MethodType, etc.
3150                                    Register recv,   // if caller wants to see it
3151                                    Register flags   // if caller wants to test it
3152                                    ) {
3153   // determine flags
3154   const Bytecodes::Code code = bytecode();
3155   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3156   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3157   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3158   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3159   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3160   const bool load_receiver       = (recv  != noreg);
3161   const bool save_flags          = (flags != noreg);
3162   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3163   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3164   assert(flags == noreg || flags == x13, "");
3165   assert(recv  == noreg || recv  == x12, "");
3166 
3167   // setup registers & access constant pool cache
3168   if (recv == noreg) {
3169     recv = x12;
3170   }
3171   if (flags == noreg) {
3172     flags = x13;
3173   }
3174   assert_different_registers(method, index, recv, flags);
3175 
3176   // save 'interpreter return address'
3177   __ save_bcp();
3178 
3179   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3180 
3181   // maybe push appendix to arguments (just before return address)
3182   if (is_invokedynamic || is_invokehandle) {
3183     Label L_no_push;
3184     __ andi(t0, flags, 1UL << ConstantPoolCacheEntry::has_appendix_shift);
3185     __ beqz(t0, L_no_push);
3186     // Push the appendix as a trailing parameter.
3187     // This must be done before we get the receiver,
3188     // since the parameter_size includes it.
3189     __ push_reg(x9);
3190     __ mv(x9, index);
3191     __ load_resolved_reference_at_index(index, x9);
3192     __ pop_reg(x9);
3193     __ push_reg(index);  // push appendix (MethodType, CallSite, etc.)
3194     __ bind(L_no_push);
3195   }
3196 
3197   // load receiver if needed (note: no return address pushed yet)
3198   if (load_receiver) {
3199     __ andi(recv, flags, ConstantPoolCacheEntry::parameter_size_mask); // parameter_size_mask = 1 << 8
3200     __ shadd(t0, recv, esp, t0, 3);
3201     __ ld(recv, Address(t0, -Interpreter::expr_offset_in_bytes(1)));
3202     __ verify_oop(recv);
3203   }
3204 
3205   // compute return type
3206   __ slli(t1, flags, XLEN - (ConstantPoolCacheEntry::tos_state_shift + ConstantPoolCacheEntry::tos_state_bits));
3207   __ srli(t1, t1, XLEN - ConstantPoolCacheEntry::tos_state_bits); // (1 << 5) - 4 --> 28~31==> t1:0~3
3208 
3209   // load return address
3210   {
3211     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3212     __ mv(t0, table_addr);
3213     __ shadd(t0, t1, t0, t1, 3);
3214     __ ld(ra, Address(t0, 0));
3215   }
3216 }
3217 
3218 void TemplateTable::invokevirtual_helper(Register index,
3219                                          Register recv,
3220                                          Register flags)
3221 {
3222   // Uses temporary registers x10, x13
3223   assert_different_registers(index, recv, x10, x13);
3224   // Test for an invoke of a final method
3225   Label notFinal;
3226   __ andi(t0, flags, 1UL << ConstantPoolCacheEntry::is_vfinal_shift);
3227   __ beqz(t0, notFinal);
3228 
3229   const Register method = index;  // method must be xmethod
3230   assert(method == xmethod, "Method must be xmethod for interpreter calling convention");
3231 
3232   // do the call - the index is actually the method to call
3233   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3234 
3235   // It's final, need a null check here!
3236   __ null_check(recv);
3237 
3238   // profile this call
3239   __ profile_final_call(x10);
3240   __ profile_arguments_type(x10, method, x14, true);
3241 
3242   __ jump_from_interpreted(method);
3243 
3244   __ bind(notFinal);
3245 
3246   // get receiver klass
3247   __ null_check(recv, oopDesc::klass_offset_in_bytes());
3248   __ load_klass(x10, recv);
3249 
3250   // profile this call
3251   __ profile_virtual_call(x10, xlocals, x13);
3252 
3253   // get target Method & entry point
3254   __ lookup_virtual_method(x10, index, method);
3255   __ profile_arguments_type(x13, method, x14, true);
3256   __ jump_from_interpreted(method);
3257 }
3258 
3259 void TemplateTable::invokevirtual(int byte_no)
3260 {
3261   transition(vtos, vtos);
3262   assert(byte_no == f2_byte, "use this argument");
3263 
3264   prepare_invoke(byte_no, xmethod, noreg, x12, x13);
3265 
3266   // xmethod: index (actually a Method*)
3267   // x12: receiver
3268   // x13: flags
3269 
3270   invokevirtual_helper(xmethod, x12, x13);
3271 }
3272 
3273 void TemplateTable::invokespecial(int byte_no)
3274 {
3275   transition(vtos, vtos);
3276   assert(byte_no == f1_byte, "use this argument");
3277 
3278   prepare_invoke(byte_no, xmethod, noreg,  // get f1 Method*
3279                  x12);  // get receiver also for null check
3280   __ verify_oop(x12);
3281   __ null_check(x12);
3282   // do the call
3283   __ profile_call(x10);
3284   __ profile_arguments_type(x10, xmethod, xbcp, false);
3285   __ jump_from_interpreted(xmethod);
3286 }
3287 
3288 void TemplateTable::invokestatic(int byte_no)
3289 {
3290   transition(vtos, vtos);
3291   assert(byte_no == f1_byte, "use this arugment");
3292 
3293   prepare_invoke(byte_no, xmethod);  // get f1 Method*
3294   // do the call
3295   __ profile_call(x10);
3296   __ profile_arguments_type(x10, xmethod, x14, false);
3297   __ jump_from_interpreted(xmethod);
3298 }
3299 
3300 void TemplateTable::fast_invokevfinal(int byte_no)
3301 {
3302   __ call_Unimplemented();
3303 }
3304 
3305 void TemplateTable::invokeinterface(int byte_no) {
3306   transition(vtos, vtos);
3307   assert(byte_no == f1_byte, "use this argument");
3308 
3309   prepare_invoke(byte_no, x10, xmethod,  // get f1 Klass*, f2 Method*
3310                  x12, x13);  // recv, flags
3311 
3312   // x10: interface klass (from f1)
3313   // xmethod: method (from f2)
3314   // x12: receiver
3315   // x13: flags
3316 
3317   // First check for Object case, then private interface method,
3318   // then regular interface method.
3319 
3320   // Special case of invokeinterface called for virtual method of
3321   // java.lang.Object. See cpCache.cpp for details
3322   Label notObjectMethod;
3323   __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_forced_virtual_shift);
3324   __ beqz(t0, notObjectMethod);
3325 
3326   invokevirtual_helper(xmethod, x12, x13);
3327   __ bind(notObjectMethod);
3328 
3329   Label no_such_interface;
3330 
3331   // Check for private method invocation - indicated by vfinal
3332   Label notVFinal;
3333   __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_vfinal_shift);
3334   __ beqz(t0, notVFinal);
3335 
3336   // Check receiver klass into x13 - also a null check
3337   __ null_check(x12, oopDesc::klass_offset_in_bytes());
3338   __ load_klass(x13, x12);
3339 
3340   Label subtype;
3341   __ check_klass_subtype(x13, x10, x14, subtype);
3342   // If we get here the typecheck failed
3343   __ j(no_such_interface);
3344   __ bind(subtype);
3345 
3346   __ profile_final_call(x10);
3347   __ profile_arguments_type(x10, xmethod, x14, true);
3348   __ jump_from_interpreted(xmethod);
3349 
3350   __ bind(notVFinal);
3351 
3352   // Get receiver klass into x13 - also a null check
3353   __ restore_locals();
3354   __ null_check(x12, oopDesc::klass_offset_in_bytes());
3355   __ load_klass(x13, x12);
3356 
3357   Label no_such_method;
3358 
3359   // Preserve method for the throw_AbstractMethodErrorVerbose.
3360   __ mv(x28, xmethod);
3361   // Receiver subtype check against REFC.
3362   // Superklass in x10. Subklass in x13. Blows t1, x30
3363   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3364                              x13, x10, noreg,
3365                              // outputs: scan temp. reg, scan temp. reg
3366                              t1, x30,
3367                              no_such_interface,
3368                              /*return_method=*/false);
3369 
3370   // profile this call
3371   __ profile_virtual_call(x13, x30, x9);
3372 
3373   // Get declaring interface class from method, and itable index
3374   __ load_method_holder(x10, xmethod);
3375   __ lwu(xmethod, Address(xmethod, Method::itable_index_offset()));
3376   __ subw(xmethod, xmethod, Method::itable_index_max);
3377   __ negw(xmethod, xmethod);
3378 
3379   // Preserve recvKlass for throw_AbstractMethodErrorVerbose
3380   __ mv(xlocals, x13);
3381   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3382                              xlocals, x10, xmethod,
3383                              // outputs: method, scan temp. reg
3384                              xmethod, x30,
3385                              no_such_interface);
3386 
3387   // xmethod: Method to call
3388   // x12: receiver
3389   // Check for abstract method error
3390   // Note: This should be done more efficiently via a throw_abstract_method_error
3391   //       interpreter entry point and a conditional jump to it in case of a null
3392   //       method.
3393   __ beqz(xmethod, no_such_method);
3394 
3395   __ profile_arguments_type(x13, xmethod, x30, true);
3396 
3397   // do the call
3398   // x12: receiver
3399   // xmethod: Method
3400   __ jump_from_interpreted(xmethod);
3401   __ should_not_reach_here();
3402 
3403   // exception handling code follows ...
3404   // note: must restore interpreter registers to canonical
3405   //       state for exception handling to work correctly!
3406 
3407   __ bind(no_such_method);
3408   // throw exception
3409   __ restore_bcp();    // bcp must be correct for exception handler   (was destroyed)
3410   __ restore_locals(); // make sure locals pointer is correct as well (was destroyed)
3411   // Pass arguments for generating a verbose error message.
3412   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodErrorVerbose), x13, x28);
3413   // the call_VM checks for exception, so we should never return here.
3414   __ should_not_reach_here();
3415 
3416   __ bind(no_such_interface);
3417   // throw exceptiong
3418   __ restore_bcp();    // bcp must be correct for exception handler   (was destroyed)
3419   __ restore_locals(); // make sure locals pointer is correct as well (was destroyed)
3420   // Pass arguments for generating a verbose error message.
3421   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3422                                      InterpreterRuntime::throw_IncompatibleClassChangeErrorVerbose), x13, x10);
3423   // the call_VM checks for exception, so we should never return here.
3424   __ should_not_reach_here();
3425   return;
3426 }
3427 
3428 void TemplateTable::invokehandle(int byte_no) {
3429   transition(vtos, vtos);
3430   assert(byte_no == f1_byte, "use this argument");
3431 
3432   prepare_invoke(byte_no, xmethod, x10, x12);
3433   __ verify_method_ptr(x12);
3434   __ verify_oop(x12);
3435   __ null_check(x12);
3436 
3437   // FIXME: profile the LambdaForm also
3438 
3439   // x30 is safe to use here as a temp reg because it is about to
3440   // be clobbered by jump_from_interpreted().
3441   __ profile_final_call(x30);
3442   __ profile_arguments_type(x30, xmethod, x14, true);
3443 
3444   __ jump_from_interpreted(xmethod);
3445 }
3446 
3447 void TemplateTable::invokedynamic(int byte_no) {
3448   transition(vtos, vtos);
3449   assert(byte_no == f1_byte, "use this argument");
3450 
3451   prepare_invoke(byte_no, xmethod, x10);
3452 
3453   // x10: CallSite object (from cpool->resolved_references[])
3454   // xmethod: MH.linkToCallSite method (from f2)
3455 
3456   // Note: x10_callsite is already pushed by prepare_invoke
3457 
3458   // %%% should make a type profile for any invokedynamic that takes a ref argument
3459   // profile this call
3460   __ profile_call(xbcp);
3461   __ profile_arguments_type(x13, xmethod, x30, false);
3462 
3463   __ verify_oop(x10);
3464 
3465   __ jump_from_interpreted(xmethod);
3466 }
3467 
3468 //-----------------------------------------------------------------------------
3469 // Allocation
3470 
3471 void TemplateTable::_new() {
3472   transition(vtos, atos);
3473 
3474   __ get_unsigned_2_byte_index_at_bcp(x13, 1);
3475   Label slow_case;
3476   Label done;
3477   Label initialize_header;
3478   Label initialize_object; // including clearing the fields
3479 
3480   __ get_cpool_and_tags(x14, x10);
3481   // Make sure the class we're about to instantiate has been resolved.
3482   // This is done before loading InstanceKlass to be consistent with the order
3483   // how Constant Pool is update (see ConstantPool::klass_at_put)
3484   const int tags_offset = Array<u1>::base_offset_in_bytes();
3485   __ add(t0, x10, x13);
3486   __ la(t0, Address(t0, tags_offset));
3487   __ membar(MacroAssembler::AnyAny);
3488   __ lbu(t0, t0);
3489   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3490   __ sub(t1, t0, (u1)JVM_CONSTANT_Class);
3491   __ bnez(t1, slow_case);
3492 
3493   // get InstanceKlass
3494   __ load_resolved_klass_at_offset(x14, x13, x14, t0);
3495 
3496   // make sure klass is initialized & doesn't have finalizer
3497   // make sure klass is fully initialized
3498   __ lbu(t0, Address(x14, InstanceKlass::init_state_offset()));
3499   __ sub(t1, t0, (u1)InstanceKlass::fully_initialized);
3500   __ bnez(t1, slow_case);
3501 
3502   // get instance_size in InstanceKlass (scaled to a count of bytes)
3503   __ lwu(x13, Address(x14, Klass::layout_helper_offset()));
3504   // test to see if it has a finalizer or is malformed in some way
3505   __ andi(t0, x13, Klass::_lh_instance_slow_path_bit);
3506   __ bnez(t0, slow_case);
3507 
3508   // Allocate the instance:
3509   //  If TLAB is enabled:
3510   //    Try to allocate in the TLAB.
3511   //    If fails, go to the slow path.
3512   //  Else If inline contiguous allocations are enabled:
3513   //    Try to allocate in eden.
3514   //    If fails due to heap end, go to slow path
3515   //
3516   //  If TLAB is enabled OR inline contiguous is enabled:
3517   //    Initialize the allocation.
3518   //    Exit.
3519   //  Go to slow path.
3520   const bool allow_shared_alloc = Universe::heap()->supports_inline_contig_alloc();
3521 
3522   if (UseTLAB) {
3523     __ tlab_allocate(x10, x13, 0, noreg, x11, slow_case);
3524 
3525     if (ZeroTLAB) {
3526       // the fields have been already cleared
3527       __ j(initialize_header);
3528     } else {
3529       // initialize both the header and fields
3530       __ j(initialize_object);
3531     }
3532   } else {
3533     // Allocation in the shared Eden, if allowed.
3534     //
3535     // x13: instance size in bytes
3536     if (allow_shared_alloc) {
3537       __ eden_allocate(x10, x13, 0, x28, slow_case);
3538     }
3539   }
3540 
3541   // If USETLAB or allow_shared_alloc are true, the object is created above and
3542   // there is an initialized need. Otherwise, skip and go to the slow path.
3543   if (UseTLAB || allow_shared_alloc) {
3544     // The object is initialized before the header. If the object size is
3545     // zero, go directly to the header initialization.
3546     __ bind(initialize_object);
3547     __ sub(x13, x13, sizeof(oopDesc));
3548     __ beqz(x13, initialize_header);
3549 
3550     // Initialize obejct fields
3551     {
3552       __ add(x12, x10, sizeof(oopDesc));
3553       Label loop;
3554       __ bind(loop);
3555       __ sd(zr, Address(x12));
3556       __ add(x12, x12, BytesPerLong);
3557       __ sub(x13, x13, BytesPerLong);
3558       __ bnez(x13, loop);
3559     }
3560 
3561     // initialize object hader only.
3562     __ bind(initialize_header);
3563     __ mv(t0, (intptr_t)markWord::prototype().value());
3564     __ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes()));
3565     __ store_klass_gap(x10, zr);   // zero klass gap for compressed oops
3566     __ store_klass(x10, x14);      // store klass last
3567 
3568     {
3569       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3570       // Trigger dtrace event for fastpath
3571       __ push(atos); // save the return value
3572       __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), x10);
3573       __ pop(atos); // restore the return value
3574     }
3575     __ j(done);
3576   }
3577 
3578   // slow case
3579   __ bind(slow_case);
3580   __ get_constant_pool(c_rarg1);
3581   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3582   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3583   __ verify_oop(x10);
3584 
3585   // continue
3586   __ bind(done);
3587   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3588   __ membar(MacroAssembler::StoreStore);
3589 }
3590 
3591 void TemplateTable::newarray() {
3592   transition(itos, atos);
3593   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3594   __ mv(c_rarg2, x10);
3595   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3596           c_rarg1, c_rarg2);
3597   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3598   __ membar(MacroAssembler::StoreStore);
3599 }
3600 
3601 void TemplateTable::anewarray() {
3602   transition(itos, atos);
3603   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3604   __ get_constant_pool(c_rarg1);
3605   __ mv(c_rarg3, x10);
3606   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3607           c_rarg1, c_rarg2, c_rarg3);
3608   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3609   __ membar(MacroAssembler::StoreStore);
3610 }
3611 
3612 void TemplateTable::arraylength() {
3613   transition(atos, itos);
3614   __ null_check(x10, arrayOopDesc::length_offset_in_bytes());
3615   __ lwu(x10, Address(x10, arrayOopDesc::length_offset_in_bytes()));
3616 }
3617 
3618 void TemplateTable::checkcast()
3619 {
3620   transition(atos, atos);
3621   Label done, is_null, ok_is_subtype, quicked, resolved;
3622   __ beqz(x10, is_null);
3623 
3624   // Get cpool & tags index
3625   __ get_cpool_and_tags(x12, x13); // x12=cpool, x13=tags array
3626   __ get_unsigned_2_byte_index_at_bcp(x9, 1); // x9=index
3627   // See if bytecode has already been quicked
3628   __ add(t0, x13, Array<u1>::base_offset_in_bytes());
3629   __ add(x11, t0, x9);
3630   __ membar(MacroAssembler::AnyAny);
3631   __ lbu(x11, x11);
3632   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3633   __ sub(t0, x11, (u1)JVM_CONSTANT_Class);
3634   __ beqz(t0, quicked);
3635 
3636   __ push(atos); // save receiver for result, and for GC
3637   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3638   // vm_result_2 has metadata result
3639   __ get_vm_result_2(x10, xthread);
3640   __ pop_reg(x13); // restore receiver
3641   __ j(resolved);
3642 
3643   // Get superklass in x10 and subklass in x13
3644   __ bind(quicked);
3645   __ mv(x13, x10); // Save object in x13; x10 needed for subtype check
3646   __ load_resolved_klass_at_offset(x12, x9, x10, t0); // x10 = klass
3647 
3648   __ bind(resolved);
3649   __ load_klass(x9, x13);
3650 
3651   // Generate subtype check.  Blows x12, x15.  Object in x13.
3652   // Superklass in x10.  Subklass in x9.
3653   __ gen_subtype_check(x9, ok_is_subtype);
3654 
3655   // Come here on failure
3656   __ push_reg(x13);
3657   // object is at TOS
3658   __ j(Interpreter::_throw_ClassCastException_entry);
3659 
3660   // Come here on success
3661   __ bind(ok_is_subtype);
3662   __ mv(x10, x13); // Restore object in x13
3663 
3664   // Collect counts on whether this test sees NULLs a lot or not.
3665   if (ProfileInterpreter) {
3666     __ j(done);
3667     __ bind(is_null);
3668     __ profile_null_seen(x12);
3669   } else {
3670     __ bind(is_null);   // same as 'done'
3671   }
3672   __ bind(done);
3673 }
3674 
3675 void TemplateTable::instanceof() {
3676   transition(atos, itos);
3677   Label done, is_null, ok_is_subtype, quicked, resolved;
3678   __ beqz(x10, is_null);
3679 
3680   // Get cpool & tags index
3681   __ get_cpool_and_tags(x12, x13); // x12=cpool, x13=tags array
3682   __ get_unsigned_2_byte_index_at_bcp(x9, 1); // x9=index
3683   // See if bytecode has already been quicked
3684   __ add(t0, x13, Array<u1>::base_offset_in_bytes());
3685   __ add(x11, t0, x9);
3686   __ membar(MacroAssembler::AnyAny);
3687   __ lbu(x11, x11);
3688   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3689   __ sub(t0, x11, (u1)JVM_CONSTANT_Class);
3690   __ beqz(t0, quicked);
3691 
3692   __ push(atos); // save receiver for result, and for GC
3693   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3694   // vm_result_2 has metadata result
3695   __ get_vm_result_2(x10, xthread);
3696   __ pop_reg(x13); // restore receiver
3697   __ verify_oop(x13);
3698   __ load_klass(x13, x13);
3699   __ j(resolved);
3700 
3701   // Get superklass in x10 and subklass in x13
3702   __ bind(quicked);
3703   __ load_klass(x13, x10);
3704   __ load_resolved_klass_at_offset(x12, x9, x10, t0);
3705 
3706   __ bind(resolved);
3707 
3708   // Generate subtype check.  Blows x12, x15
3709   // Superklass in x10.  Subklass in x13.
3710   __ gen_subtype_check(x13, ok_is_subtype);
3711 
3712   // Come here on failure
3713   __ mv(x10, zr);
3714   __ j(done);
3715   // Come here on success
3716   __ bind(ok_is_subtype);
3717   __ li(x10, 1);
3718 
3719   // Collect counts on whether this test sees NULLs a lot or not.
3720   if (ProfileInterpreter) {
3721     __ j(done);
3722     __ bind(is_null);
3723     __ profile_null_seen(x12);
3724   } else {
3725     __ bind(is_null);   // same as 'done'
3726   }
3727   __ bind(done);
3728   // x10 = 0: obj == NULL or  obj is not an instanceof the specified klass
3729   // x10 = 1: obj != NULL and obj is     an instanceof the specified klass
3730 }
3731 
3732 //-----------------------------------------------------------------------------
3733 // Breakpoints
3734 void TemplateTable::_breakpoint() {
3735   // Note: We get here even if we are single stepping..
3736   // jbug inists on setting breakpoints at every bytecode
3737   // even if we are in single step mode.
3738 
3739   transition(vtos, vtos);
3740 
3741   // get the unpatched byte code
3742   __ get_method(c_rarg1);
3743   __ call_VM(noreg,
3744              CAST_FROM_FN_PTR(address,
3745                               InterpreterRuntime::get_original_bytecode_at),
3746              c_rarg1, xbcp);
3747   __ mv(x9, x10);
3748 
3749   // post the breakpoint event
3750   __ call_VM(noreg,
3751              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
3752              xmethod, xbcp);
3753 
3754   // complete the execution of original bytecode
3755   __ mv(t0, x9);
3756   __ dispatch_only_normal(vtos);
3757 }
3758 
3759 //-----------------------------------------------------------------------------
3760 // Exceptions
3761 
3762 void TemplateTable::athrow() {
3763   transition(atos, vtos);
3764   __ null_check(x10);
3765   __ j(Interpreter::throw_exception_entry());
3766 }
3767 
3768 //-----------------------------------------------------------------------------
3769 // Synchronization
3770 //
3771 // Note: monitorenter & exit are symmetric routines; which is reflected
3772 //       in the assembly code structure as well
3773 //
3774 // Stack layout:
3775 //
3776 // [expressions  ] <--- esp               = expression stack top
3777 // ..
3778 // [expressions  ]
3779 // [monitor entry] <--- monitor block top = expression stack bot
3780 // ..
3781 // [monitor entry]
3782 // [frame data   ] <--- monitor block bot
3783 // ...
3784 // [saved fp     ] <--- fp
3785 void TemplateTable::monitorenter()
3786 {
3787   transition(atos, vtos);
3788 
3789    // check for NULL object
3790    __ null_check(x10);
3791 
3792    const Address monitor_block_top(
3793          fp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3794    const Address monitor_block_bot(
3795          fp, frame::interpreter_frame_initial_sp_offset * wordSize);
3796    const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3797 
3798    Label allocated;
3799 
3800    // initialize entry pointer
3801    __ mv(c_rarg1, zr); // points to free slot or NULL
3802 
3803    // find a free slot in the monitor block (result in c_rarg1)
3804    {
3805      Label entry, loop, exit, notUsed;
3806      __ ld(c_rarg3, monitor_block_top); // points to current entry,
3807                                         // starting with top-most entry
3808      __ la(c_rarg2, monitor_block_bot); // points to word before bottom
3809 
3810      __ j(entry);
3811 
3812      __ bind(loop);
3813      // check if current entry is used
3814      // if not used then remember entry in c_rarg1
3815      __ ld(t0, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
3816      __ bnez(t0, notUsed);
3817      __ mv(c_rarg1, c_rarg3);
3818      __ bind(notUsed);
3819      // check if current entry is for same object
3820      // if same object then stop searching
3821      __ beq(x10, t0, exit);
3822      // otherwise advance to next entry
3823      __ add(c_rarg3, c_rarg3, entry_size);
3824      __ bind(entry);
3825      // check if bottom reached
3826      // if not at bottom then check this entry
3827      __ bne(c_rarg3, c_rarg2, loop);
3828      __ bind(exit);
3829    }
3830 
3831    __ bnez(c_rarg1, allocated); // check if a slot has been found and
3832                              // if found, continue with that on
3833 
3834    // allocate one if there's no free slot
3835    {
3836      Label entry, loop;
3837      // 1. compute new pointers            // esp: old expression stack top
3838      __ ld(c_rarg1, monitor_block_bot);    // c_rarg1: old expression stack bottom
3839      __ sub(esp, esp, entry_size);         // move expression stack top
3840      __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
3841      __ mv(c_rarg3, esp);                  // set start value for copy loop
3842      __ sd(c_rarg1, monitor_block_bot);    // set new monitor block bottom
3843      __ sub(sp, sp, entry_size);           // make room for the monitor
3844 
3845      __ j(entry);
3846      // 2. move expression stack contents
3847      __ bind(loop);
3848      __ ld(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
3849                                                    // word from old location
3850      __ sd(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
3851      __ add(c_rarg3, c_rarg3, wordSize);           // advance to next word
3852      __ bind(entry);
3853      __ bne(c_rarg3, c_rarg1, loop);    // check if bottom reached.if not at bottom
3854                                         // then copy next word
3855    }
3856 
3857    // call run-time routine
3858    // c_rarg1: points to monitor entry
3859    __ bind(allocated);
3860 
3861    // Increment bcp to point to the next bytecode, so exception
3862    // handling for async. exceptions work correctly.
3863    // The object has already been poped from the stack, so the
3864    // expression stack looks correct.
3865    __ addi(xbcp, xbcp, 1);
3866 
3867    // store object
3868    __ sd(x10, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3869    __ lock_object(c_rarg1);
3870 
3871    // check to make sure this monitor doesn't cause stack overflow after locking
3872    __ save_bcp();  // in case of exception
3873    __ generate_stack_overflow_check(0);
3874 
3875    // The bcp has already been incremented. Just need to dispatch to
3876    // next instruction.
3877    __ dispatch_next(vtos);
3878 }
3879 
3880 void TemplateTable::monitorexit()
3881 {
3882   transition(atos, vtos);
3883 
3884   // check for NULL object
3885   __ null_check(x10);
3886 
3887   const Address monitor_block_top(
3888         fp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3889   const Address monitor_block_bot(
3890         fp, frame::interpreter_frame_initial_sp_offset * wordSize);
3891   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3892 
3893   Label found;
3894 
3895   // find matching slot
3896   {
3897     Label entry, loop;
3898     __ ld(c_rarg1, monitor_block_top); // points to current entry,
3899                                         // starting with top-most entry
3900     __ la(c_rarg2, monitor_block_bot); // points to word before bottom
3901                                         // of monitor block
3902     __ j(entry);
3903 
3904     __ bind(loop);
3905     // check if current entry is for same object
3906     __ ld(t0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3907     // if same object then stop searching
3908     __ beq(x10, t0, found);
3909     // otherwise advance to next entry
3910     __ add(c_rarg1, c_rarg1, entry_size);
3911     __ bind(entry);
3912     // check if bottom reached
3913     // if not at bottom then check this entry
3914     __ bne(c_rarg1, c_rarg2, loop);
3915   }
3916 
3917   // error handling. Unlocking was not block-structured
3918   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3919                    InterpreterRuntime::throw_illegal_monitor_state_exception));
3920   __ should_not_reach_here();
3921 
3922   // call run-time routine
3923   __ bind(found);
3924   __ push_ptr(x10); // make sure object is on stack (contract with oopMaps)
3925   __ unlock_object(c_rarg1);
3926   __ pop_ptr(x10); // discard object
3927 }
3928 
3929 // Wide instructions
3930 void TemplateTable::wide()
3931 {
3932   __ load_unsigned_byte(x9, at_bcp(1));
3933   __ mv(t0, (address)Interpreter::_wentry_point);
3934   __ shadd(t0, x9, t0, t1, 3);
3935   __ ld(t0, Address(t0));
3936   __ jr(t0);
3937 }
3938 
3939 // Multi arrays
3940 void TemplateTable::multianewarray() {
3941   transition(vtos, atos);
3942   __ load_unsigned_byte(x10, at_bcp(3)); // get number of dimensions
3943   // last dim is on top of stack; we want address of first one:
3944   // first_addr = last_addr + (ndims - 1) * wordSize
3945   __ shadd(c_rarg1, x10, esp, c_rarg1, 3);
3946   __ sub(c_rarg1, c_rarg1, wordSize);
3947   call_VM(x10,
3948           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
3949           c_rarg1);
3950   __ load_unsigned_byte(x11, at_bcp(3));
3951   __ shadd(esp, x11, esp, t0, 3);
3952 }