1 /*
   2  * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.inline.hpp"
  29 #include "gc/shared/barrierSetAssembler.hpp"
  30 #include "gc/shared/collectedHeap.hpp"
  31 #include "gc/shared/tlab_globals.hpp"
  32 #include "interpreter/interp_masm.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "interpreter/interpreterRuntime.hpp"
  35 #include "interpreter/templateTable.hpp"
  36 #include "memory/universe.hpp"
  37 #include "oops/method.hpp"
  38 #include "oops/methodData.hpp"
  39 #include "oops/objArrayKlass.hpp"
  40 #include "oops/oop.inline.hpp"
  41 #include "prims/jvmtiExport.hpp"
  42 #include "prims/methodHandles.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 #include "runtime/synchronizer.hpp"
  47 #include "utilities/powerOfTwo.hpp"
  48 
  49 #define __ _masm->
  50 
  51 // Address computation: local variables
  52 
  53 static inline Address iaddress(int n) {
  54   return Address(xlocals, Interpreter::local_offset_in_bytes(n));
  55 }
  56 
  57 static inline Address laddress(int n) {
  58   return iaddress(n + 1);
  59 }
  60 
  61 static inline Address faddress(int n) {
  62   return iaddress(n);
  63 }
  64 
  65 static inline Address daddress(int n) {
  66   return laddress(n);
  67 }
  68 
  69 static inline Address aaddress(int n) {
  70   return iaddress(n);
  71 }
  72 
  73 static inline Address iaddress(Register r,  Register temp, InterpreterMacroAssembler* _masm) {
  74   assert_cond(_masm != NULL);
  75   _masm->slli(temp, r, 3);
  76   _masm->add(temp, xlocals, temp);
  77   return Address(temp, 0);
  78 }
  79 
  80 static inline Address laddress(Register r, Register temp,
  81                                InterpreterMacroAssembler* _masm) {
  82   assert_cond(_masm != NULL);
  83   _masm->slli(temp, r, 3);
  84   _masm->add(temp, xlocals, temp);
  85   return Address(temp, Interpreter::local_offset_in_bytes(1));;
  86 }
  87 
  88 static inline Address faddress(Register r, Register temp, InterpreterMacroAssembler* _masm) {
  89   return iaddress(r, temp, _masm);
  90 }
  91 
  92 static inline Address daddress(Register r, Register temp,
  93                                InterpreterMacroAssembler* _masm) {
  94   return laddress(r, temp, _masm);
  95 }
  96 
  97 static inline Address aaddress(Register r, Register temp, InterpreterMacroAssembler* _masm) {
  98   return iaddress(r, temp, _masm);
  99 }
 100 
 101 static inline Address at_rsp() {
 102   return Address(esp, 0);
 103 }
 104 
 105 // At top of Java expression stack which may be different than esp().  It
 106 // isn't for category 1 objects.
 107 static inline Address at_tos   () {
 108   return Address(esp,  Interpreter::expr_offset_in_bytes(0));
 109 }
 110 
 111 static inline Address at_tos_p1() {
 112   return Address(esp,  Interpreter::expr_offset_in_bytes(1));
 113 }
 114 
 115 static inline Address at_tos_p2() {
 116   return Address(esp,  Interpreter::expr_offset_in_bytes(2));
 117 }
 118 
 119 static inline Address at_tos_p3() {
 120   return Address(esp,  Interpreter::expr_offset_in_bytes(3));
 121 }
 122 
 123 static inline Address at_tos_p4() {
 124   return Address(esp,  Interpreter::expr_offset_in_bytes(4));
 125 }
 126 
 127 static inline Address at_tos_p5() {
 128   return Address(esp,  Interpreter::expr_offset_in_bytes(5));
 129 }
 130 
 131 // Miscelaneous helper routines
 132 // Store an oop (or NULL) at the Address described by obj.
 133 // If val == noreg this means store a NULL
 134 static void do_oop_store(InterpreterMacroAssembler* _masm,
 135                          Address dst,
 136                          Register val,
 137                          DecoratorSet decorators) {
 138   assert(val == noreg || val == x10, "parameter is just for looks");
 139   assert_cond(_masm != NULL);
 140   __ store_heap_oop(dst, val, x29, x11, decorators);
 141 }
 142 
 143 static void do_oop_load(InterpreterMacroAssembler* _masm,
 144                         Address src,
 145                         Register dst,
 146                         DecoratorSet decorators) {
 147   assert_cond(_masm != NULL);
 148   __ load_heap_oop(dst, src, x7, x11, decorators);
 149 }
 150 
 151 Address TemplateTable::at_bcp(int offset) {
 152   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 153   return Address(xbcp, offset);
 154 }
 155 
 156 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 157                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 158                                    int byte_no)
 159 {
 160   if (!RewriteBytecodes)  { return; }
 161   Label L_patch_done;
 162 
 163   switch (bc) {
 164     case Bytecodes::_fast_aputfield:  // fall through
 165     case Bytecodes::_fast_bputfield:  // fall through
 166     case Bytecodes::_fast_zputfield:  // fall through
 167     case Bytecodes::_fast_cputfield:  // fall through
 168     case Bytecodes::_fast_dputfield:  // fall through
 169     case Bytecodes::_fast_fputfield:  // fall through
 170     case Bytecodes::_fast_iputfield:  // fall through
 171     case Bytecodes::_fast_lputfield:  // fall through
 172     case Bytecodes::_fast_sputfield: {
 173       // We skip bytecode quickening for putfield instructions when
 174       // the put_code written to the constant pool cache is zero.
 175       // This is required so that every execution of this instruction
 176       // calls out to InterpreterRuntime::resolve_get_put to do
 177       // additional, required work.
 178       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 179       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 180       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 181       __ mv(bc_reg, bc);
 182       __ beqz(temp_reg, L_patch_done);
 183       break;
 184     }
 185     default:
 186       assert(byte_no == -1, "sanity");
 187       // the pair bytecodes have already done the load.
 188       if (load_bc_into_bc_reg) {
 189         __ mv(bc_reg, bc);
 190       }
 191   }
 192 
 193   if (JvmtiExport::can_post_breakpoint()) {
 194     Label L_fast_patch;
 195     // if a breakpoint is present we can't rewrite the stream directly
 196     __ load_unsigned_byte(temp_reg, at_bcp(0));
 197     __ addi(temp_reg, temp_reg, -Bytecodes::_breakpoint); // temp_reg is temporary register.
 198     __ bnez(temp_reg, L_fast_patch);
 199     // Let breakpoint table handling rewrite to quicker bytecode
 200     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), xmethod, xbcp, bc_reg);
 201     __ j(L_patch_done);
 202     __ bind(L_fast_patch);
 203   }
 204 
 205 #ifdef ASSERT
 206   Label L_okay;
 207   __ load_unsigned_byte(temp_reg, at_bcp(0));
 208   __ beq(temp_reg, bc_reg, L_okay);
 209   __ addi(temp_reg, temp_reg, -(int) Bytecodes::java_code(bc));
 210   __ beqz(temp_reg, L_okay);
 211   __ stop("patching the wrong bytecode");
 212   __ bind(L_okay);
 213 #endif
 214 
 215   // patch bytecode
 216   __ sb(bc_reg, at_bcp(0));
 217   __ bind(L_patch_done);
 218 }
 219 
 220 // Individual instructions
 221 
 222 void TemplateTable::nop() {
 223   transition(vtos, vtos);
 224   // nothing to do
 225 }
 226 
 227 void TemplateTable::shouldnotreachhere() {
 228   transition(vtos, vtos);
 229   __ stop("should not reach here bytecode");
 230 }
 231 
 232 void TemplateTable::aconst_null()
 233 {
 234   transition(vtos, atos);
 235   __ mv(x10, zr);
 236 }
 237 
 238 void TemplateTable::iconst(int value)
 239 {
 240   transition(vtos, itos);
 241   __ li(x10, value);
 242 }
 243 
 244 void TemplateTable::lconst(int value)
 245 {
 246   transition(vtos, ltos);
 247   __ li(x10, value);
 248 }
 249 
 250 void TemplateTable::fconst(int value)
 251 {
 252   transition(vtos, ftos);
 253   static float fBuf[2] = {1.0, 2.0};
 254   __ mv(t0, (intptr_t)fBuf);
 255   switch (value) {
 256     case 0:
 257       __ fmv_w_x(f10, zr);
 258       break;
 259     case 1:
 260       __ flw(f10, t0, 0);
 261       break;
 262     case 2:
 263       __ flw(f10, t0, sizeof(float));
 264       break;
 265     default:
 266       ShouldNotReachHere();
 267   }
 268 }
 269 
 270 void TemplateTable::dconst(int value)
 271 {
 272   transition(vtos, dtos);
 273   static double dBuf[2] = {1.0, 2.0};
 274   __ mv(t0, (intptr_t)dBuf);
 275   switch (value) {
 276     case 0:
 277       __ fmv_d_x(f10, zr);
 278       break;
 279     case 1:
 280       __ fld(f10, t0, 0);
 281       break;
 282     case 2:
 283       __ fld(f10, t0, sizeof(double));
 284       break;
 285     default:
 286       ShouldNotReachHere();
 287   }
 288 }
 289 
 290 void TemplateTable::bipush()
 291 {
 292   transition(vtos, itos);
 293   __ load_signed_byte(x10, at_bcp(1));
 294 }
 295 
 296 void TemplateTable::sipush()
 297 {
 298   transition(vtos, itos);
 299   __ load_unsigned_short(x10, at_bcp(1));
 300   __ grevw(x10, x10);
 301   __ sraiw(x10, x10, 16);
 302 }
 303 
 304 void TemplateTable::ldc(bool wide)
 305 {
 306   transition(vtos, vtos);
 307   Label call_ldc, notFloat, notClass, notInt, Done;
 308 
 309   if (wide) {
 310    __ get_unsigned_2_byte_index_at_bcp(x11, 1);
 311   } else {
 312    __ load_unsigned_byte(x11, at_bcp(1));
 313   }
 314   __ get_cpool_and_tags(x12, x10);
 315 
 316   const int base_offset = ConstantPool::header_size() * wordSize;
 317   const int tags_offset = Array<u1>::base_offset_in_bytes();
 318 
 319   // get type
 320   __ addi(x13, x11, tags_offset);
 321   __ add(x13, x10, x13);
 322   __ membar(MacroAssembler::AnyAny);
 323   __ lbu(x13, Address(x13, 0));
 324   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
 325 
 326   // unresolved class - get the resolved class
 327   __ mv(t1, (u1)JVM_CONSTANT_UnresolvedClass);
 328   __ beq(x13, t1, call_ldc);
 329 
 330   // unresolved class in error state - call into runtime to throw the error
 331   // from the first resolution attempt
 332   __ mv(t1, (u1)JVM_CONSTANT_UnresolvedClassInError);
 333   __ beq(x13, t1, call_ldc);
 334 
 335   // resolved class - need to call vm to get java mirror of the class
 336   __ mv(t1, (u1)JVM_CONSTANT_Class);
 337   __ bne(x13, t1, notClass);
 338 
 339   __ bind(call_ldc);
 340   __ mv(c_rarg1, wide);
 341   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 342   __ push_ptr(x10);
 343   __ verify_oop(x10);
 344   __ j(Done);
 345 
 346   __ bind(notClass);
 347   __ mv(t1, (u1)JVM_CONSTANT_Float);
 348   __ bne(x13, t1, notFloat);
 349 
 350   // ftos
 351   __ slli(x11, x11, 3);
 352   __ add(x11, x12, x11);
 353   __ flw(f10, Address(x11, base_offset));
 354   __ push_f(f10);
 355   __ j(Done);
 356 
 357   __ bind(notFloat);
 358 
 359   __ mv(t1, (u1)JVM_CONSTANT_Integer);
 360   __ bne(x13, t1, notInt);
 361 
 362   // itos
 363   __ slli(x11, x11, 3);
 364   __ add(x11, x12, x11);
 365   __ lw(x10, Address(x11, base_offset));
 366   __ push_i(x10);
 367   __ j(Done);
 368 
 369   __ bind(notInt);
 370   condy_helper(Done);
 371 
 372   __ bind(Done);
 373 }
 374 
 375 // Fast path for caching oop constants.
 376 void TemplateTable::fast_aldc(bool wide)
 377 {
 378   transition(vtos, atos);
 379 
 380   const Register result = x10;
 381   const Register tmp = x11;
 382   const Register rarg = x12;
 383 
 384   const int index_size = wide ? sizeof(u2) : sizeof(u1);
 385 
 386   Label resolved;
 387 
 388   // We are resolved if the resolved reference cache entry contains a
 389   // non-null object (String, MethodType, etc.)
 390   assert_different_registers(result, tmp);
 391   __ get_cache_index_at_bcp(tmp, 1, index_size);
 392   __ load_resolved_reference_at_index(result, tmp);
 393   __ bnez(result, resolved);
 394 
 395   const address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 396 
 397   // first time invocation - must resolve first
 398   __ mv(rarg, (int)bytecode());
 399   __ call_VM(result, entry, rarg);
 400 
 401   __ bind(resolved);
 402 
 403   { // Check for the null sentinel.
 404     // If we just called the VM, it already did the mapping for us,
 405     // but it's harmless to retry.
 406     Label notNull;
 407 
 408     // Stash null_sentinel address to get its value later
 409     int32_t offset = 0;
 410     __ movptr_with_offset(rarg, Universe::the_null_sentinel_addr(), offset);
 411     __ ld(tmp, Address(rarg, offset));
 412     __ resolve_oop_handle(tmp);
 413     __ bne(result, tmp, notNull);
 414     __ mv(result, zr);  // NULL object reference
 415     __ bind(notNull);
 416   }
 417 
 418   if (VerifyOops) {
 419     // Safe to call with 0 result
 420     __ verify_oop(result);
 421   }
 422 }
 423 
 424 void TemplateTable::ldc2_w()
 425 {
 426     transition(vtos, vtos);
 427     Label notDouble, notLong, Done;
 428     __ get_unsigned_2_byte_index_at_bcp(x10, 1);
 429 
 430     __ get_cpool_and_tags(x11, x12);
 431     const int base_offset = ConstantPool::header_size() * wordSize;
 432     const int tags_offset = Array<u1>::base_offset_in_bytes();
 433 
 434     // get type
 435     __ add(x12, x12, x10);
 436     __ load_unsigned_byte(x12, Address(x12, tags_offset));
 437     __ mv(t1, JVM_CONSTANT_Double);
 438     __ bne(x12, t1, notDouble);
 439 
 440     // dtos
 441     __ slli(x12, x10, 3);
 442     __ add(x12, x11, x12);
 443     __ fld(f10, Address(x12, base_offset));
 444     __ push_d(f10);
 445     __ j(Done);
 446 
 447     __ bind(notDouble);
 448     __ mv(t1, (int)JVM_CONSTANT_Long);
 449     __ bne(x12, t1, notLong);
 450 
 451     // ltos
 452     __ slli(x10, x10, 3);
 453     __ add(x10, x11, x10);
 454     __ ld(x10, Address(x10, base_offset));
 455     __ push_l(x10);
 456     __ j(Done);
 457 
 458     __ bind(notLong);
 459     condy_helper(Done);
 460     __ bind(Done);
 461 
 462 }
 463 
 464 void TemplateTable::condy_helper(Label& Done)
 465 {
 466   const Register obj = x10;
 467   const Register rarg = x11;
 468   const Register flags = x12;
 469   const Register off = x13;
 470 
 471   const address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 472 
 473   __ mv(rarg, (int) bytecode());
 474   __ call_VM(obj, entry, rarg);
 475 
 476   __ get_vm_result_2(flags, xthread);
 477 
 478   // VMr = obj = base address to find primitive value to push
 479   // VMr2 = flags = (tos, off) using format of CPCE::_flags
 480   __ mv(off, flags);
 481   __ mv(t0, ConstantPoolCacheEntry::field_index_mask);
 482   __ andrw(off, off, t0);
 483 
 484   __ add(off, obj, off);
 485   const Address field(off, 0); // base + R---->base + offset
 486 
 487   __ slli(flags, flags, registerSize - (ConstantPoolCacheEntry::tos_state_shift + ConstantPoolCacheEntry::tos_state_bits));
 488   __ srli(flags, flags, registerSize - ConstantPoolCacheEntry::tos_state_bits); // (1 << 5) - 4 --> 28~31==> flags:0~3
 489 
 490   switch (bytecode()) {
 491     case Bytecodes::_ldc:   // fall through
 492     case Bytecodes::_ldc_w: {
 493       // tos in (itos, ftos, stos, btos, ctos, ztos)
 494       Label notInt, notFloat, notShort, notByte, notChar, notBool;
 495       __ mv(t1, itos);
 496       __ bne(flags, t1, notInt);
 497       // itos
 498       __ lw(x10, field);
 499       __ push(itos);
 500       __ j(Done);
 501 
 502       __ bind(notInt);
 503       __ mv(t1, ftos);
 504       __ bne(flags, t1, notFloat);
 505       // ftos
 506       __ load_float(field);
 507       __ push(ftos);
 508       __ j(Done);
 509 
 510       __ bind(notFloat);
 511       __ mv(t1, stos);
 512       __ bne(flags, t1, notShort);
 513       // stos
 514       __ load_signed_short(x10, field);
 515       __ push(stos);
 516       __ j(Done);
 517 
 518       __ bind(notShort);
 519       __ mv(t1, btos);
 520       __ bne(flags, t1, notByte);
 521       // btos
 522       __ load_signed_byte(x10, field);
 523       __ push(btos);
 524       __ j(Done);
 525 
 526       __ bind(notByte);
 527       __ mv(t1, ctos);
 528       __ bne(flags, t1, notChar);
 529       // ctos
 530       __ load_unsigned_short(x10, field);
 531       __ push(ctos);
 532       __ j(Done);
 533 
 534       __ bind(notChar);
 535       __ mv(t1, ztos);
 536       __ bne(flags, t1, notBool);
 537       // ztos
 538       __ load_signed_byte(x10, field);
 539       __ push(ztos);
 540       __ j(Done);
 541 
 542       __ bind(notBool);
 543       break;
 544     }
 545 
 546     case Bytecodes::_ldc2_w: {
 547       Label notLong, notDouble;
 548       __ mv(t1, ltos);
 549       __ bne(flags, t1, notLong);
 550       // ltos
 551       __ ld(x10, field);
 552       __ push(ltos);
 553       __ j(Done);
 554 
 555       __ bind(notLong);
 556       __ mv(t1, dtos);
 557       __ bne(flags, t1, notDouble);
 558       // dtos
 559       __ load_double(field);
 560       __ push(dtos);
 561       __ j(Done);
 562 
 563       __ bind(notDouble);
 564       break;
 565     }
 566 
 567     default:
 568       ShouldNotReachHere();
 569   }
 570 
 571   __ stop("bad ldc/condy");
 572 }
 573 
 574 void TemplateTable::locals_index(Register reg, int offset)
 575 {
 576   __ lbu(reg, at_bcp(offset));
 577   __ neg(reg, reg);
 578 }
 579 
 580 void TemplateTable::iload() {
 581   iload_internal();
 582 }
 583 
 584 void TemplateTable::nofast_iload() {
 585   iload_internal(may_not_rewrite);
 586 }
 587 
 588 void TemplateTable::iload_internal(RewriteControl rc) {
 589   transition(vtos, itos);
 590   if (RewriteFrequentPairs && rc == may_rewrite) {
 591     Label rewrite, done;
 592     const Register bc = x14;
 593 
 594     // get next bytecode
 595     __ load_unsigned_byte(x11, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 596 
 597     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 598     // last two iloads in a pair.  Comparing against fast_iload means that
 599     // the next bytecode is neither an iload or a caload, and therefore
 600     // an iload pair.
 601     __ mv(t1, Bytecodes::_iload);
 602     __ beq(x11, t1, done);
 603 
 604     // if _fast_iload rewrite to _fast_iload2
 605     __ mv(t1, Bytecodes::_fast_iload);
 606     __ mv(bc, Bytecodes::_fast_iload2);
 607     __ beq(x11, t1, rewrite);
 608 
 609     // if _caload rewrite to _fast_icaload
 610     __ mv(t1, Bytecodes::_caload);
 611     __ mv(bc, Bytecodes::_fast_icaload);
 612     __ beq(x11, t1, rewrite);
 613 
 614     // else rewrite to _fast_iload
 615     __ mv(bc, Bytecodes::_fast_iload);
 616 
 617     // rewrite
 618     // bc: new bytecode
 619     __ bind(rewrite);
 620     patch_bytecode(Bytecodes::_iload, bc, x11, false);
 621     __ bind(done);
 622 
 623   }
 624 
 625   // do iload, get the local value into tos
 626   locals_index(x11);
 627   __ lw(x10, iaddress(x11, x10, _masm));
 628 }
 629 
 630 void TemplateTable::fast_iload2()
 631 {
 632   transition(vtos, itos);
 633   locals_index(x11);
 634   __ lw(x10, iaddress(x11, x10, _masm));
 635   __ push(itos);
 636   locals_index(x11, 3);
 637   __ lw(x10, iaddress(x11, x10, _masm));
 638 }
 639 
 640 void TemplateTable::fast_iload()
 641 {
 642   transition(vtos, itos);
 643   locals_index(x11);
 644   __ lw(x10, iaddress(x11, x10, _masm));
 645 }
 646 
 647 void TemplateTable::lload()
 648 {
 649   transition(vtos, ltos);
 650   __ lbu(x11, at_bcp(1));
 651   __ slli(x11, x11, LogBytesPerWord);
 652   __ sub(x11, xlocals, x11);
 653   __ ld(x10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 654 }
 655 
 656 void TemplateTable::fload()
 657 {
 658   transition(vtos, ftos);
 659   locals_index(x11);
 660   __ flw(f10, faddress(x11, t0, _masm));
 661 }
 662 
 663 void TemplateTable::dload()
 664 {
 665   transition(vtos, dtos);
 666   __ lbu(x11, at_bcp(1));
 667   __ slli(x11, x11, LogBytesPerWord);
 668   __ sub(x11, xlocals, x11);
 669   __ fld(f10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 670 }
 671 
 672 void TemplateTable::aload()
 673 {
 674   transition(vtos, atos);
 675   locals_index(x11);
 676   __ ld(x10, iaddress(x11, x10, _masm));
 677 
 678 }
 679 
 680 void TemplateTable::locals_index_wide(Register reg) {
 681   __ lhu(reg, at_bcp(2));
 682   __ grevhu(reg, reg); // reverse bytes in half-word and zero-extend
 683   __ neg(reg, reg);
 684 }
 685 
 686 void TemplateTable::wide_iload() {
 687   transition(vtos, itos);
 688   locals_index_wide(x11);
 689   __ lw(x10, iaddress(x11, t0, _masm));
 690 }
 691 
 692 void TemplateTable::wide_lload()
 693 {
 694   transition(vtos, ltos);
 695   __ lhu(x11, at_bcp(2));
 696   __ grevhu(x11, x11); // reverse bytes in half-word and zero-extend
 697   __ slli(x11, x11, LogBytesPerWord);
 698   __ sub(x11, xlocals, x11);
 699   __ ld(x10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 700 }
 701 
 702 void TemplateTable::wide_fload()
 703 {
 704   transition(vtos, ftos);
 705   locals_index_wide(x11);
 706   __ flw(f10, faddress(x11, t0, _masm));
 707 }
 708 
 709 void TemplateTable::wide_dload()
 710 {
 711   transition(vtos, dtos);
 712   __ lhu(x11, at_bcp(2));
 713   __ grevhu(x11, x11); // reverse bytes in half-word and zero-extend
 714   __ slli(x11, x11, LogBytesPerWord);
 715   __ sub(x11, xlocals, x11);
 716   __ fld(f10, Address(x11, Interpreter::local_offset_in_bytes(1)));
 717 }
 718 
 719 void TemplateTable::wide_aload()
 720 {
 721   transition(vtos, atos);
 722   locals_index_wide(x11);
 723   __ ld(x10, aaddress(x11, t0, _masm));
 724 }
 725 
 726 void TemplateTable::index_check(Register array, Register index)
 727 {
 728   // destroys x11, t0
 729   // check array
 730   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
 731   // sign extend index for use by indexed load
 732   // check index
 733   const Register length = t0;
 734   __ lwu(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 735   if (index != x11) {
 736     assert(x11 != array, "different registers");
 737     __ mv(x11, index);
 738   }
 739   Label ok;
 740   __ addw(index, index, zr);
 741   __ bltu(index, length, ok);
 742   __ mv(x13, array);
 743   __ mv(t0, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 744   __ jr(t0);
 745   __ bind(ok);
 746 }
 747 
 748 void TemplateTable::iaload()
 749 {
 750   transition(itos, itos);
 751   __ mv(x11, x10);
 752   __ pop_ptr(x10);
 753   // x10: array
 754   // x11: index
 755   index_check(x10, x11); // leaves index in x11
 756   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
 757   __ slli(t0, x11, 2);
 758   __ add(x10, t0, x10);
 759   __ access_load_at(T_INT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 760   __ addw(x10, x10, zr); // signed extended
 761 }
 762 
 763 void TemplateTable::laload()
 764 {
 765   transition(itos, ltos);
 766   __ mv(x11, x10);
 767   __ pop_ptr(x10);
 768   // x10: array
 769   // x11: index
 770   index_check(x10, x11); // leaves index in x11
 771   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
 772   __ slli(t0, x11, 3);
 773   __ add(x10, t0, x10);
 774   __ access_load_at(T_LONG, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 775 }
 776 
 777 void TemplateTable::faload()
 778 {
 779   transition(itos, ftos);
 780   __ mv(x11, x10);
 781   __ pop_ptr(x10);
 782   // x10: array
 783   // x11: index
 784   index_check(x10, x11); // leaves index in x11
 785   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
 786   __ slli(t0, x11, 2);
 787   __ add(x10, t0, x10);
 788   __ access_load_at(T_FLOAT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 789 }
 790 
 791 void TemplateTable::daload()
 792 {
 793   transition(itos, dtos);
 794   __ mv(x11, x10);
 795   __ pop_ptr(x10);
 796   // x10: array
 797   // x11: index
 798   index_check(x10, x11); // leaves index in x11
 799   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
 800   __ slli(t0, x11, 3);
 801   __ add(x10, t0, x10);
 802   __ access_load_at(T_DOUBLE, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 803 }
 804 
 805 void TemplateTable::aaload()
 806 {
 807   transition(itos, atos);
 808   __ mv(x11, x10);
 809   __ pop_ptr(x10);
 810   // x10: array
 811   // x11: index
 812   index_check(x10, x11); // leaves index in x11
 813   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 814   __ slli(t0, x11, LogBytesPerHeapOop);
 815   __ add(x10, t0, x10);
 816   do_oop_load(_masm,
 817               Address(x10),
 818               x10,
 819               IS_ARRAY);
 820 }
 821 
 822 void TemplateTable::baload()
 823 {
 824   transition(itos, itos);
 825   __ mv(x11, x10);
 826   __ pop_ptr(x10);
 827   // x10: array
 828   // x11: index
 829   index_check(x10, x11); // leaves index in x11
 830   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
 831   __ slli(t0, x11, 0);
 832   __ add(x10, t0, x10);
 833   __ access_load_at(T_BYTE, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 834 }
 835 
 836 void TemplateTable::caload()
 837 {
 838  transition(itos, itos);
 839   __ mv(x11, x10);
 840   __ pop_ptr(x10);
 841   // x10: array
 842   // x11: index
 843   index_check(x10, x11); // leaves index in x11
 844   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
 845   __ slli(t0, x11, 1);
 846   __ add(x10, t0, x10);
 847   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 848 }
 849 
 850 // iload followed by caload frequent pair
 851 void TemplateTable::fast_icaload()
 852 {
 853   transition(vtos, itos);
 854   // load index out of locals
 855   locals_index(x12);
 856   __ lw(x11, iaddress(x12, x11, _masm));
 857   __ pop_ptr(x10);
 858 
 859   // x10: array
 860   // x11: index
 861   index_check(x10, x11); // leaves index in x11, kills t0
 862   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1); // addi, max imm is 2^11
 863   __ slli(t0, x11, 1);
 864   __ add(x10, x10, t0);
 865   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 866 }
 867 
 868 void TemplateTable::saload()
 869 {
 870   transition(itos, itos);
 871   __ mv(x11, x10);
 872   __ pop_ptr(x10);
 873   // x10: array
 874   // x11: index
 875   index_check(x10, x11); // leaves index in x11, kills t0
 876   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_SHORT) >> 1);
 877   __ slli(t0, x11, 1);
 878   __ add(x10, t0, x10);
 879   __ access_load_at(T_SHORT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg);
 880 }
 881 
 882 void TemplateTable::iload(int n)
 883 {
 884   transition(vtos, itos);
 885   __ lw(x10, iaddress(n));
 886 }
 887 
 888 void TemplateTable::lload(int n)
 889 {
 890   transition(vtos, ltos);
 891   __ ld(x10, laddress(n));
 892 }
 893 
 894 void TemplateTable::fload(int n)
 895 {
 896   transition(vtos, ftos);
 897   __ flw(f10, faddress(n));
 898 }
 899 
 900 void TemplateTable::dload(int n)
 901 {
 902   transition(vtos, dtos);
 903   __ fld(f10, daddress(n));
 904 }
 905 
 906 void TemplateTable::aload(int n)
 907 {
 908   transition(vtos, atos);
 909   __ ld(x10, iaddress(n));
 910 }
 911 
 912 void TemplateTable::aload_0() {
 913   aload_0_internal();
 914 }
 915 
 916 void TemplateTable::nofast_aload_0() {
 917   aload_0_internal(may_not_rewrite);
 918 }
 919 
 920 void TemplateTable::aload_0_internal(RewriteControl rc) {
 921   // According to bytecode histograms, the pairs:
 922   //
 923   // _aload_0, _fast_igetfield
 924   // _aload_0, _fast_agetfield
 925   // _aload_0, _fast_fgetfield
 926   //
 927   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 928   // _aload_0 bytecode checks if the next bytecode is either
 929   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 930   // rewrites the current bytecode into a pair bytecode; otherwise it
 931   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 932   // the pair check anymore.
 933   //
 934   // Note: If the next bytecode is _getfield, the rewrite must be
 935   //       delayed, otherwise we may miss an opportunity for a pair.
 936   //
 937   // Also rewrite frequent pairs
 938   //   aload_0, aload_1
 939   //   aload_0, iload_1
 940   // These bytecodes with a small amount of code are most profitable
 941   // to rewrite
 942   if (RewriteFrequentPairs && rc == may_rewrite) {
 943     Label rewrite, done;
 944     const Register bc = x14;
 945 
 946     // get next bytecode
 947     __ load_unsigned_byte(x11, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 948 
 949     // if _getfield then wait with rewrite
 950     __ mv(t1, Bytecodes::Bytecodes::_getfield);
 951     __ beq(x11, t1, done);
 952 
 953     // if _igetfield then rewrite to _fast_iaccess_0
 954     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 955     __ mv(t1, Bytecodes::_fast_igetfield);
 956     __ mv(bc, Bytecodes::_fast_iaccess_0);
 957     __ beq(x11, t1, rewrite);
 958 
 959     // if _agetfield then rewrite to _fast_aaccess_0
 960     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 961     __ mv(t1, Bytecodes::_fast_agetfield);
 962     __ mv(bc, Bytecodes::_fast_aaccess_0);
 963     __ beq(x11, t1, rewrite);
 964 
 965     // if _fgetfield then rewrite to _fast_faccess_0
 966     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 967     __ mv(t1, Bytecodes::_fast_fgetfield);
 968     __ mv(bc, Bytecodes::_fast_faccess_0);
 969     __ beq(x11, t1, rewrite);
 970 
 971     // else rewrite to _fast_aload0
 972     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 973     __ mv(bc, Bytecodes::Bytecodes::_fast_aload_0);
 974 
 975     // rewrite
 976     // bc: new bytecode
 977     __ bind(rewrite);
 978     patch_bytecode(Bytecodes::_aload_0, bc, x11, false);
 979 
 980     __ bind(done);
 981   }
 982 
 983   // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
 984   aload(0);
 985 }
 986 
 987 void TemplateTable::istore()
 988 {
 989   transition(itos, vtos);
 990   locals_index(x11);
 991   __ sw(x10, iaddress(x11, t0, _masm));
 992 }
 993 
 994 void TemplateTable::lstore()
 995 {
 996   transition(ltos, vtos);
 997   locals_index(x11);
 998   __ sd(x10, laddress(x11, t0, _masm));
 999 }
1000 
1001 void TemplateTable::fstore() {
1002   transition(ftos, vtos);
1003   locals_index(x11);
1004   __ fsw(f10, iaddress(x11, t0, _masm));
1005 }
1006 
1007 void TemplateTable::dstore() {
1008   transition(dtos, vtos);
1009   locals_index(x11);
1010   __ fsd(f10, daddress(x11, t0, _masm));
1011 }
1012 
1013 void TemplateTable::astore()
1014 {
1015   transition(vtos, vtos);
1016   __ pop_ptr(x10);
1017   locals_index(x11);
1018   __ sd(x10, aaddress(x11, t0, _masm));
1019 }
1020 
1021 void TemplateTable::wide_istore() {
1022   transition(vtos, vtos);
1023   __ pop_i();
1024   locals_index_wide(x11);
1025   __ sw(x10, iaddress(x11, t0, _masm));
1026 }
1027 
1028 void TemplateTable::wide_lstore() {
1029   transition(vtos, vtos);
1030   __ pop_l();
1031   locals_index_wide(x11);
1032   __ sd(x10, laddress(x11, t0, _masm));
1033 }
1034 
1035 void TemplateTable::wide_fstore() {
1036   transition(vtos, vtos);
1037   __ pop_f();
1038   locals_index_wide(x11);
1039   __ fsw(f10, faddress(x11, t0, _masm));
1040 }
1041 
1042 void TemplateTable::wide_dstore() {
1043   transition(vtos, vtos);
1044   __ pop_d();
1045   locals_index_wide(x11);
1046   __ fsd(f10, daddress(x11, t0, _masm));
1047 }
1048 
1049 void TemplateTable::wide_astore() {
1050   transition(vtos, vtos);
1051   __ pop_ptr(x10);
1052   locals_index_wide(x11);
1053   __ sd(x10, aaddress(x11, t0, _masm));
1054 }
1055 
1056 void TemplateTable::iastore() {
1057   transition(itos, vtos);
1058   __ pop_i(x11);
1059   __ pop_ptr(x13);
1060   // x10: value
1061   // x11: index
1062   // x13: array
1063   index_check(x13, x11); // prefer index in x11
1064   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
1065   __ slli(t0, x11, 2);
1066   __ add(t0, x13, t0);
1067   __ access_store_at(T_INT, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg);
1068 }
1069 
1070 void TemplateTable::lastore() {
1071   transition(ltos, vtos);
1072   __ pop_i(x11);
1073   __ pop_ptr(x13);
1074   // x10: value
1075   // x11: index
1076   // x13: array
1077   index_check(x13, x11); // prefer index in x11
1078   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3);
1079   __ slli(t0, x11, 3);
1080   __ add(t0, x13, t0);
1081   __ access_store_at(T_LONG, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg);
1082 }
1083 
1084 void TemplateTable::fastore() {
1085   transition(ftos, vtos);
1086   __ pop_i(x11);
1087   __ pop_ptr(x13);
1088   // f10: value
1089   // x11:  index
1090   // x13:  array
1091   index_check(x13, x11); // prefer index in x11
1092   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
1093   __ slli(t0, x11, 2);
1094   __ add(t0, x13, t0);
1095   __ access_store_at(T_FLOAT, IN_HEAP | IS_ARRAY, Address(t0, 0), noreg /* ftos */, noreg, noreg);
1096 }
1097 
1098 void TemplateTable::dastore() {
1099   transition(dtos, vtos);
1100   __ pop_i(x11);
1101   __ pop_ptr(x13);
1102   // f10: value
1103   // x11:  index
1104   // x13:  array
1105   index_check(x13, x11); // prefer index in x11
1106   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
1107   __ slli(t0, x11, 3);
1108   __ add(t0, x13, t0);
1109   __ access_store_at(T_DOUBLE, IN_HEAP | IS_ARRAY, Address(t0, 0), noreg /* dtos */, noreg, noreg);
1110 }
1111 
1112 void TemplateTable::aastore() {
1113   Label is_null, ok_is_subtype, done;
1114   transition(vtos, vtos);
1115   // stack: ..., array, index, value
1116   __ ld(x10, at_tos());    // value
1117   __ ld(x12, at_tos_p1()); // index
1118   __ ld(x13, at_tos_p2()); // array
1119 
1120   index_check(x13, x12);     // kills x11
1121   __ add(x14, x12, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
1122   __ slli(x14, x14, LogBytesPerHeapOop);
1123   __ add(x14, x13, x14);
1124 
1125   Address element_address(x14, 0);
1126 
1127   // do array store check - check for NULL value first
1128   __ beqz(x10, is_null);
1129 
1130   // Move subklass into x11
1131   __ load_klass(x11, x10);
1132   // Move superklass into x10
1133   __ load_klass(x10, x13);
1134   __ ld(x10, Address(x10,
1135                      ObjArrayKlass::element_klass_offset()));
1136   // Compress array + index * oopSize + 12 into a single register.  Frees x12.
1137 
1138   // Generate subtype check.  Blows x12, x15
1139   // Superklass in x10.  Subklass in x11.
1140   __ gen_subtype_check(x11, ok_is_subtype); //todo
1141 
1142   // Come here on failure
1143   // object is at TOS
1144   __ j(Interpreter::_throw_ArrayStoreException_entry);
1145 
1146   // Come here on success
1147   __ bind(ok_is_subtype);
1148 
1149   // Get the value we will store
1150   __ ld(x10, at_tos());
1151   // Now store using the appropriate barrier
1152   do_oop_store(_masm, element_address, x10, IS_ARRAY);
1153   __ j(done);
1154 
1155   // Have a NULL in x10, x13=array, x12=index.  Store NULL at ary[idx]
1156   __ bind(is_null);
1157   __ profile_null_seen(x12);
1158 
1159   // Store a NULL
1160   do_oop_store(_masm, element_address, noreg, IS_ARRAY);
1161 
1162   // Pop stack arguments
1163   __ bind(done);
1164   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1165 
1166 }
1167 
1168 void TemplateTable::bastore()
1169 {
1170   transition(itos, vtos);
1171   __ pop_i(x11);
1172   __ pop_ptr(x13);
1173   // x10: value
1174   // x11: index
1175   // x13: array
1176   index_check(x13, x11); // prefer index in x11
1177 
1178   // Need to check whether array is boolean or byte
1179   // since both types share the bastore bytecode.
1180   __ load_klass(x12, x13);
1181   __ lwu(x12, Address(x12, Klass::layout_helper_offset()));
1182   Label L_skip;
1183   __ andi(t0, x12, Klass::layout_helper_boolean_diffbit());
1184   __ beqz(t0, L_skip);
1185   __ andi(x10, x10, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1186   __ bind(L_skip);
1187 
1188   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
1189 
1190   __ add(x11, x13, x11);
1191   __ access_store_at(T_BYTE, IN_HEAP | IS_ARRAY, Address(x11, 0), x10, noreg, noreg);
1192 }
1193 
1194 void TemplateTable::castore()
1195 {
1196   transition(itos, vtos);
1197   __ pop_i(x11);
1198   __ pop_ptr(x13);
1199   // x10: value
1200   // x11: index
1201   // x13: array
1202   index_check(x13, x11); // prefer index in x11
1203   __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1);
1204   __ slli(t0, x11, 1);
1205   __ add(t0, x13, t0);
1206   __ access_store_at(T_CHAR, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg);
1207 }
1208 
1209 void TemplateTable::sastore()
1210 {
1211   castore();
1212 }
1213 
1214 void TemplateTable::istore(int n)
1215 {
1216   transition(itos, vtos);
1217   __ sd(x10, iaddress(n));
1218 }
1219 
1220 void TemplateTable::lstore(int n)
1221 {
1222   transition(ltos, vtos);
1223   __ sd(x10, laddress(n));
1224 }
1225 
1226 void TemplateTable::fstore(int n)
1227 {
1228   transition(ftos, vtos);
1229   __ fsw(f10, faddress(n));
1230 }
1231 
1232 void TemplateTable::dstore(int n)
1233 {
1234   transition(dtos, vtos);
1235   __ fsd(f10, daddress(n));
1236 }
1237 
1238 void TemplateTable::astore(int n)
1239 {
1240   transition(vtos, vtos);
1241   __ pop_ptr(x10);
1242   __ sd(x10, iaddress(n));
1243 }
1244 
1245 void TemplateTable::pop()
1246 {
1247   transition(vtos, vtos);
1248   __ addi(esp, esp, Interpreter::stackElementSize);
1249 }
1250 
1251 void TemplateTable::pop2()
1252 {
1253   transition(vtos, vtos);
1254   __ addi(esp, esp, 2 * Interpreter::stackElementSize);
1255 }
1256 
1257 void TemplateTable::dup()
1258 {
1259   transition(vtos, vtos);
1260   __ ld(x10, Address(esp, 0));
1261   __ push_reg(x10);
1262   // stack: ..., a, a
1263 }
1264 
1265 void TemplateTable::dup_x1()
1266 {
1267   transition(vtos, vtos);
1268   // stack: ..., a, b
1269   __ ld(x10, at_tos());  // load b
1270   __ ld(x12, at_tos_p1());  // load a
1271   __ sd(x10, at_tos_p1());  // store b
1272   __ sd(x12, at_tos());  // store a
1273   __ push_reg(x10);                  // push b
1274   // stack: ..., b, a, b
1275 }
1276 
1277 void TemplateTable::dup_x2()
1278 {
1279   transition(vtos, vtos);
1280   // stack: ..., a, b, c
1281   __ ld(x10, at_tos());  // load c
1282   __ ld(x12, at_tos_p2());  // load a
1283   __ sd(x10, at_tos_p2());  // store c in a
1284   __ push_reg(x10);      // push c
1285   // stack: ..., c, b, c, c
1286   __ ld(x10, at_tos_p2());  // load b
1287   __ sd(x12, at_tos_p2());  // store a in b
1288   // stack: ..., c, a, c, c
1289   __ sd(x10, at_tos_p1());  // store b in c
1290   // stack: ..., c, a, b, c
1291 }
1292 
1293 void TemplateTable::dup2()
1294 {
1295   transition(vtos, vtos);
1296   // stack: ..., a, b
1297   __ ld(x10, at_tos_p1());  // load a
1298   __ push_reg(x10);                  // push a
1299   __ ld(x10, at_tos_p1());  // load b
1300   __ push_reg(x10);                  // push b
1301   // stack: ..., a, b, a, b
1302 }
1303 
1304 void TemplateTable::dup2_x1()
1305 {
1306   transition(vtos, vtos);
1307   // stack: ..., a, b, c
1308   __ ld(x12, at_tos());     // load c
1309   __ ld(x10, at_tos_p1());  // load b
1310   __ push_reg(x10);             // push b
1311   __ push_reg(x12);             // push c
1312   // stack: ..., a, b, c, b, c
1313   __ sd(x12, at_tos_p3());  // store c in b
1314   // stack: ..., a, c, c, b, c
1315   __ ld(x12, at_tos_p4());  // load a
1316   __ sd(x12, at_tos_p2());  // store a in 2nd c
1317   // stack: ..., a, c, a, b, c
1318   __ sd(x10, at_tos_p4());  // store b in a
1319   // stack: ..., b, c, a, b, c
1320 }
1321 
1322 void TemplateTable::dup2_x2()
1323 {
1324   transition(vtos, vtos);
1325   // stack: ..., a, b, c, d
1326   __ ld(x12, at_tos());     // load d
1327   __ ld(x10, at_tos_p1());  // load c
1328   __ push_reg(x10);             // push c
1329   __ push_reg(x12);             // push d
1330   // stack: ..., a, b, c, d, c, d
1331   __ ld(x10, at_tos_p4());  // load b
1332   __ sd(x10, at_tos_p2());  // store b in d
1333   __ sd(x12, at_tos_p4());  // store d in b
1334   // stack: ..., a, d, c, b, c, d
1335   __ ld(x12, at_tos_p5());  // load a
1336   __ ld(x10, at_tos_p3());  // load c
1337   __ sd(x12, at_tos_p3());  // store a in c
1338   __ sd(x10, at_tos_p5());  // store c in a
1339   // stack: ..., c, d, a, b, c, d
1340 }
1341 
1342 void TemplateTable::swap()
1343 {
1344   transition(vtos, vtos);
1345   // stack: ..., a, b
1346   __ ld(x12, at_tos_p1());  // load a
1347   __ ld(x10, at_tos());     // load b
1348   __ sd(x12, at_tos());     // store a in b
1349   __ sd(x10, at_tos_p1());  // store b in a
1350   // stack: ..., b, a
1351 }
1352 
1353 void TemplateTable::iop2(Operation op)
1354 {
1355   transition(itos, itos);
1356   // x10 <== x11 op x10
1357   __ pop_i(x11);
1358   switch (op) {
1359     case add  : __ addw(x10, x11, x10);  break;
1360     case sub  : __ subw(x10, x11, x10);  break;
1361     case mul  : __ mulw(x10, x11, x10);  break;
1362     case _and : __ andrw(x10, x11, x10); break;
1363     case _or  : __ orrw(x10, x11, x10);  break;
1364     case _xor : __ xorrw(x10, x11, x10); break;
1365     case shl  : __ sllw(x10, x11, x10);  break;
1366     case shr  : __ sraw(x10, x11, x10);  break;
1367     case ushr : __ srlw(x10, x11, x10);  break;
1368     default   : ShouldNotReachHere();
1369   }
1370 }
1371 
1372 void TemplateTable::lop2(Operation op)
1373 {
1374   transition(ltos, ltos);
1375   // x10 <== x11 op x10
1376   __ pop_l(x11);
1377   switch (op) {
1378     case add  : __ add(x10, x11, x10);  break;
1379     case sub  : __ sub(x10, x11, x10);  break;
1380     case mul  : __ mul(x10, x11, x10);  break;
1381     case _and : __ andr(x10, x11, x10); break;
1382     case _or  : __ orr(x10, x11, x10);  break;
1383     case _xor : __ xorr(x10, x11, x10); break;
1384     default   : ShouldNotReachHere();
1385   }
1386 }
1387 
1388 void TemplateTable::idiv()
1389 {
1390   transition(itos, itos);
1391   // explicitly check for div0
1392   Label no_div0;
1393   __ bnez(x10, no_div0);
1394   __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
1395   __ jr(t0);
1396   __ bind(no_div0);
1397   __ pop_i(x11);
1398   // x10 <== x11 idiv x10
1399   __ corrected_idivl(x10, x11, x10, /* want_remainder */ false);
1400 }
1401 
1402 void TemplateTable::irem()
1403 {
1404   transition(itos, itos);
1405   // explicitly check for div0
1406   Label no_div0;
1407   __ bnez(x10, no_div0);
1408   __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
1409   __ jr(t0);
1410   __ bind(no_div0);
1411   __ pop_i(x11);
1412   // x10 <== x11 irem x10
1413   __ corrected_idivl(x10, x11, x10, /* want_remainder */ true);
1414 }
1415 
1416 void TemplateTable::lmul()
1417 {
1418   transition(ltos, ltos);
1419   __ pop_l(x11);
1420   __ mul(x10, x10, x11);
1421 }
1422 
1423 void TemplateTable::ldiv()
1424 {
1425   transition(ltos, ltos);
1426   // explicitly check for div0
1427   Label no_div0;
1428   __ bnez(x10, no_div0);
1429   __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
1430   __ jr(t0);
1431   __ bind(no_div0);
1432   __ pop_l(x11);
1433   // x10 <== x11 ldiv x10
1434   __ corrected_idivq(x10, x11, x10, /* want_remainder */ false);
1435 }
1436 
1437 void TemplateTable::lrem()
1438 {
1439   transition(ltos, ltos);
1440   // explicitly check for div0
1441   Label no_div0;
1442   __ bnez(x10, no_div0);
1443   __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
1444   __ jr(t0);
1445   __ bind(no_div0);
1446   __ pop_l(x11);
1447   // x10 <== x11 lrem x10
1448   __ corrected_idivq(x10, x11, x10, /* want_remainder */ true);
1449 }
1450 
1451 void TemplateTable::lshl()
1452 {
1453   transition(itos, ltos);
1454   // shift count is in x10
1455   __ pop_l(x11);
1456   __ sll(x10, x11, x10);
1457 }
1458 
1459 void TemplateTable::lshr()
1460 {
1461   transition(itos, ltos);
1462   // shift count is in x10
1463   __ pop_l(x11);
1464   __ sra(x10, x11, x10);
1465 }
1466 
1467 void TemplateTable::lushr()
1468 {
1469   transition(itos, ltos);
1470   // shift count is in x10
1471   __ pop_l(x11);
1472   __ srl(x10, x11, x10);
1473 }
1474 
1475 void TemplateTable::fop2(Operation op)
1476 {
1477   transition(ftos, ftos);
1478   switch (op) {
1479     case add:
1480       __ pop_f(f11);
1481       __ fadd_s(f10, f11, f10);
1482       break;
1483     case sub:
1484       __ pop_f(f11);
1485       __ fsub_s(f10, f11, f10);
1486       break;
1487     case mul:
1488       __ pop_f(f11);
1489       __ fmul_s(f10, f11, f10);
1490       break;
1491     case div:
1492       __ pop_f(f11);
1493       __ fdiv_s(f10, f11, f10);
1494       break;
1495     case rem:
1496       __ fmv_s(f11, f10);
1497       __ pop_f(f10);
1498       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem));
1499       break;
1500     default:
1501       ShouldNotReachHere();
1502   }
1503 }
1504 
1505 void TemplateTable::dop2(Operation op)
1506 {
1507   transition(dtos, dtos);
1508   switch (op) {
1509     case add:
1510       __ pop_d(f11);
1511       __ fadd_d(f10, f11, f10);
1512       break;
1513     case sub:
1514       __ pop_d(f11);
1515       __ fsub_d(f10, f11, f10);
1516       break;
1517     case mul:
1518       __ pop_d(f11);
1519       __ fmul_d(f10, f11, f10);
1520       break;
1521     case div:
1522       __ pop_d(f11);
1523       __ fdiv_d(f10, f11, f10);
1524       break;
1525     case rem:
1526       __ fmv_d(f11, f10);
1527       __ pop_d(f10);
1528       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem));
1529       break;
1530     default:
1531       ShouldNotReachHere();
1532   }
1533 }
1534 
1535 void TemplateTable::ineg()
1536 {
1537   transition(itos, itos);
1538   __ negw(x10, x10);
1539 }
1540 
1541 void TemplateTable::lneg()
1542 {
1543   transition(ltos, ltos);
1544   __ neg(x10, x10);
1545 }
1546 
1547 void TemplateTable::fneg()
1548 {
1549   transition(ftos, ftos);
1550   __ fneg_s(f10, f10);
1551 }
1552 
1553 void TemplateTable::dneg()
1554 {
1555   transition(dtos, dtos);
1556   __ fneg_d(f10, f10);
1557 }
1558 
1559 void TemplateTable::iinc()
1560 {
1561   transition(vtos, vtos);
1562   __ load_signed_byte(x11, at_bcp(2)); // get constant
1563   locals_index(x12);
1564   __ ld(x10, iaddress(x12, x10, _masm));
1565   __ addw(x10, x10, x11);
1566   __ sd(x10, iaddress(x12, t0, _masm));
1567 }
1568 
1569 void TemplateTable::wide_iinc()
1570 {
1571   transition(vtos, vtos);
1572   __ lwu(x11, at_bcp(2)); // get constant and index
1573   __ grev16wu(x11, x11); // reverse bytes in half-word (32bit) and zero-extend
1574   __ zero_ext(x12, x11, 48);
1575   __ neg(x12, x12);
1576   __ slli(x11, x11, 32);
1577   __ srai(x11, x11, 48);
1578   __ ld(x10, iaddress(x12, t0, _masm));
1579   __ addw(x10, x10, x11);
1580   __ sd(x10, iaddress(x12, t0, _masm));
1581 }
1582 
1583 void TemplateTable::convert()
1584 {
1585   // Checking
1586 #ifdef ASSERT
1587   {
1588     TosState tos_in  = ilgl;
1589     TosState tos_out = ilgl;
1590     switch (bytecode()) {
1591       case Bytecodes::_i2l: // fall through
1592       case Bytecodes::_i2f: // fall through
1593       case Bytecodes::_i2d: // fall through
1594       case Bytecodes::_i2b: // fall through
1595       case Bytecodes::_i2c: // fall through
1596       case Bytecodes::_i2s: tos_in = itos; break;
1597       case Bytecodes::_l2i: // fall through
1598       case Bytecodes::_l2f: // fall through
1599       case Bytecodes::_l2d: tos_in = ltos; break;
1600       case Bytecodes::_f2i: // fall through
1601       case Bytecodes::_f2l: // fall through
1602       case Bytecodes::_f2d: tos_in = ftos; break;
1603       case Bytecodes::_d2i: // fall through
1604       case Bytecodes::_d2l: // fall through
1605       case Bytecodes::_d2f: tos_in = dtos; break;
1606       default             : ShouldNotReachHere();
1607     }
1608     switch (bytecode()) {
1609       case Bytecodes::_l2i: // fall through
1610       case Bytecodes::_f2i: // fall through
1611       case Bytecodes::_d2i: // fall through
1612       case Bytecodes::_i2b: // fall through
1613       case Bytecodes::_i2c: // fall through
1614       case Bytecodes::_i2s: tos_out = itos; break;
1615       case Bytecodes::_i2l: // fall through
1616       case Bytecodes::_f2l: // fall through
1617       case Bytecodes::_d2l: tos_out = ltos; break;
1618       case Bytecodes::_i2f: // fall through
1619       case Bytecodes::_l2f: // fall through
1620       case Bytecodes::_d2f: tos_out = ftos; break;
1621       case Bytecodes::_i2d: // fall through
1622       case Bytecodes::_l2d: // fall through
1623       case Bytecodes::_f2d: tos_out = dtos; break;
1624       default             : ShouldNotReachHere();
1625     }
1626     transition(tos_in, tos_out);
1627   }
1628 #endif // ASSERT
1629 
1630   // Conversion
1631   switch (bytecode()) {
1632     case Bytecodes::_i2l:
1633       __ sign_ext(x10, x10, registerSize - 32);
1634       break;
1635     case Bytecodes::_i2f:
1636       __ fcvt_s_w(f10, x10);
1637       break;
1638     case Bytecodes::_i2d:
1639       __ fcvt_d_w(f10, x10);
1640       break;
1641     case Bytecodes::_i2b:
1642       __ sign_ext(x10, x10, registerSize - 8);
1643       break;
1644     case Bytecodes::_i2c:
1645       __ zero_ext(x10, x10, registerSize - 16);
1646       break;
1647     case Bytecodes::_i2s:
1648       __ sign_ext(x10, x10, registerSize - 16);
1649       break;
1650     case Bytecodes::_l2i:
1651       __ addw(x10, x10, zr);
1652       break;
1653     case Bytecodes::_l2f:
1654       __ fcvt_s_l(f10, x10);
1655       break;
1656     case Bytecodes::_l2d:
1657       __ fcvt_d_l(f10, x10);
1658       break;
1659     case Bytecodes::_f2i:
1660       __ fcvt_w_s_safe(x10, f10);
1661       break;
1662     case Bytecodes::_f2l:
1663       __ fcvt_l_s_safe(x10, f10);
1664       break;
1665     case Bytecodes::_f2d:
1666       __ fcvt_d_s(f10, f10);
1667       break;
1668     case Bytecodes::_d2i:
1669       __ fcvt_w_d_safe(x10, f10);
1670       break;
1671     case Bytecodes::_d2l:
1672       __ fcvt_l_d_safe(x10, f10);
1673       break;
1674     case Bytecodes::_d2f:
1675       __ fcvt_s_d(f10, f10);
1676       break;
1677     default:
1678       ShouldNotReachHere();
1679   }
1680 }
1681 
1682 void TemplateTable::lcmp()
1683 {
1684   transition(ltos, itos);
1685   __ pop_l(x11);
1686   __ cmp_l2i(t0, x11, x10);
1687   __ mv(x10, t0);
1688 }
1689 
1690 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1691 {
1692   // For instruction feq, flt and fle, the result is 0 if either operand is NaN
1693   if (is_float) {
1694     __ pop_f(f11);
1695     // if unordered_result < 0:
1696     //   we want -1 for unordered or less than, 0 for equal and 1 for
1697     //   greater than.
1698     // else:
1699     //   we want -1 for less than, 0 for equal and 1 for unordered or
1700     //   greater than.
1701     // f11 primary, f10 secondary
1702     __ float_compare(x10, f11, f10, unordered_result);
1703   } else {
1704     __ pop_d(f11);
1705     // if unordered_result < 0:
1706     //   we want -1 for unordered or less than, 0 for equal and 1 for
1707     //   greater than.
1708     // else:
1709     //   we want -1 for less than, 0 for equal and 1 for unordered or
1710     //   greater than.
1711     // f11 primary, f10 secondary
1712     __ double_compare(x10, f11, f10, unordered_result);
1713   }
1714 }
1715 
1716 void TemplateTable::branch(bool is_jsr, bool is_wide)
1717 {
1718   // We might be moving to a safepoint.  The thread which calls
1719   // Interpreter::notice_safepoints() will effectively flush its cache
1720   // when it makes a system call, but we need to do something to
1721   // ensure that we see the changed dispatch table.
1722   __ membar(MacroAssembler::LoadLoad);
1723 
1724   __ profile_taken_branch(x10, x11);
1725   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1726                              InvocationCounter::counter_offset();
1727   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1728                               InvocationCounter::counter_offset();
1729 
1730   // load branch displacement
1731   if (!is_wide) {
1732     __ lhu(x12, at_bcp(1));
1733     __ grevh(x12, x12); // reverse bytes in half-word and sign-extend
1734   } else {
1735     __ lwu(x12, at_bcp(1));
1736     __ grevw(x12, x12); // reverse bytes in word and sign-extend
1737   }
1738 
1739   // Handle all the JSR stuff here, then exit.
1740   // It's much shorter and cleaner than intermingling with the non-JSR
1741   // normal-branch stuff occurring below.
1742 
1743   if (is_jsr) {
1744     // compute return address as bci
1745     __ ld(t1, Address(xmethod, Method::const_offset()));
1746     __ add(t1, t1,
1747            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1748     __ sub(x11, xbcp, t1);
1749     __ push_i(x11);
1750     // Adjust the bcp by the 16-bit displacement in x12
1751     __ add(xbcp, xbcp, x12);
1752     __ load_unsigned_byte(t0, Address(xbcp, 0));
1753     // load the next target bytecode into t0, it is the argument of dispatch_only
1754     __ dispatch_only(vtos, /*generate_poll*/true);
1755     return;
1756   }
1757 
1758   // Normal (non-jsr) branch handling
1759 
1760   // Adjust the bcp by the displacement in x12
1761   __ add(xbcp, xbcp, x12);
1762 
1763   assert(UseLoopCounter || !UseOnStackReplacement,
1764          "on-stack-replacement requires loop counters");
1765   Label backedge_counter_overflow;
1766   Label dispatch;
1767   if (UseLoopCounter) {
1768     // increment backedge counter for backward branches
1769     // x10: MDO
1770     // x11: MDO bumped taken-count
1771     // x12: target offset
1772     __ bgtz(x12, dispatch); // count only if backward branch
1773 
1774     // check if MethodCounters exists
1775     Label has_counters;
1776     __ ld(t0, Address(xmethod, Method::method_counters_offset()));
1777     __ bnez(t0, has_counters);
1778     __ push_reg(x10);
1779     __ push_reg(x11);
1780     __ push_reg(x12);
1781     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1782             InterpreterRuntime::build_method_counters), xmethod);
1783     __ pop_reg(x12);
1784     __ pop_reg(x11);
1785     __ pop_reg(x10);
1786     __ ld(t0, Address(xmethod, Method::method_counters_offset()));
1787     __ beqz(t0, dispatch); // No MethodCounters allocated, OutOfMemory
1788     __ bind(has_counters);
1789 
1790     Label no_mdo;
1791     int increment = InvocationCounter::count_increment;
1792     if (ProfileInterpreter) {
1793       // Are we profiling?
1794       __ ld(x11, Address(xmethod, in_bytes(Method::method_data_offset())));
1795       __ beqz(x11, no_mdo);
1796       // Increment the MDO backedge counter
1797       const Address mdo_backedge_counter(x11, in_bytes(MethodData::backedge_counter_offset()) +
1798                                          in_bytes(InvocationCounter::counter_offset()));
1799       const Address mask(x11, in_bytes(MethodData::backedge_mask_offset()));
1800       __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1801                                  x10, t0, false,
1802                                  UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1803       __ j(dispatch);
1804     }
1805     __ bind(no_mdo);
1806     // Increment backedge counter in MethodCounters*
1807     __ ld(t0, Address(xmethod, Method::method_counters_offset()));
1808     const Address mask(t0, in_bytes(MethodCounters::backedge_mask_offset()));
1809     __ increment_mask_and_jump(Address(t0, be_offset), increment, mask,
1810                                x10, t1, false,
1811                                UseOnStackReplacement ? &backedge_counter_overflow : &dispatch);
1812     __ bind(dispatch);
1813   }
1814 
1815   // Pre-load the next target bytecode into t0
1816   __ load_unsigned_byte(t0, Address(xbcp, 0));
1817 
1818   // continue with the bytecode @ target
1819   // t0: target bytecode
1820   // xbcp: target bcp
1821   __ dispatch_only(vtos, /*generate_poll*/true);
1822 
1823   if (UseLoopCounter && UseOnStackReplacement) {
1824     // invocation counter overflow
1825     __ bind(backedge_counter_overflow);
1826     __ neg(x12, x12);
1827     __ add(x12, x12, xbcp);     // branch xbcp
1828     // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
1829     __ call_VM(noreg,
1830                CAST_FROM_FN_PTR(address,
1831                                 InterpreterRuntime::frequency_counter_overflow),
1832                x12);
1833     __ load_unsigned_byte(x11, Address(xbcp, 0));  // restore target bytecode
1834 
1835     // x10: osr nmethod (osr ok) or NULL (osr not possible)
1836     // w11: target bytecode
1837     // x12: temporary
1838     __ beqz(x10, dispatch);     // test result -- no osr if null
1839     // nmethod may have been invalidated (VM may block upon call_VM return)
1840     __ lbu(x12, Address(x10, nmethod::state_offset()));
1841     if (nmethod::in_use != 0) {
1842       __ sub(x12, x12, nmethod::in_use);
1843     }
1844     __ bnez(x12, dispatch);
1845 
1846     // We have the address of an on stack replacement routine in x10
1847     // We need to prepare to execute the OSR method. First we must
1848     // migrate the locals and monitors off of the stack.
1849 
1850     __ mv(x9, x10);                             // save the nmethod
1851 
1852     call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
1853 
1854     // x10 is OSR buffer, move it to expected parameter location
1855     __ mv(j_rarg0, x10);
1856 
1857     // remove activation
1858     // get sender esp
1859     __ ld(esp,
1860         Address(fp, frame::interpreter_frame_sender_sp_offset * wordSize));
1861     // remove frame anchor
1862     __ leave();
1863     // Ensure compiled code always sees stack at proper alignment
1864     __ andi(sp, esp, -16);
1865 
1866     // and begin the OSR nmethod
1867     __ ld(t0, Address(x9, nmethod::osr_entry_point_offset()));
1868     __ jr(t0);
1869   }
1870 }
1871 
1872 void TemplateTable::if_0cmp(Condition cc)
1873 {
1874   transition(itos, vtos);
1875   // assume branch is more often taken than not (loops use backward branches)
1876   Label not_taken;
1877 
1878   __ addw(x10, x10, zr);
1879   switch (cc) {
1880     case equal:
1881       __ bnez(x10, not_taken);
1882       break;
1883     case not_equal:
1884       __ beqz(x10, not_taken);
1885       break;
1886     case less:
1887       __ bgez(x10, not_taken);
1888       break;
1889     case less_equal:
1890       __ bgtz(x10, not_taken);
1891       break;
1892     case greater:
1893       __ blez(x10, not_taken);
1894       break;
1895     case greater_equal:
1896       __ bltz(x10, not_taken);
1897       break;
1898     default:
1899       break;
1900   }
1901 
1902   branch(false, false);
1903   __ bind(not_taken);
1904   __ profile_not_taken_branch(x10);
1905 }
1906 
1907 void TemplateTable::if_icmp(Condition cc)
1908 {
1909   transition(itos, vtos);
1910   // assume branch is more often taken than not (loops use backward branches)
1911   Label not_taken;
1912   __ pop_i(x11);
1913   __ addw(x10, x10, zr);
1914   switch (cc) {
1915     case equal:
1916       __ bne(x11, x10, not_taken);
1917       break;
1918     case not_equal:
1919       __ beq(x11, x10, not_taken);
1920       break;
1921     case less:
1922       __ bge(x11, x10, not_taken);
1923       break;
1924     case less_equal:
1925       __ bgt(x11, x10, not_taken);
1926       break;
1927     case greater:
1928       __ ble(x11, x10, not_taken);
1929       break;
1930     case greater_equal:
1931       __ blt(x11, x10, not_taken);
1932       break;
1933     default:
1934       break;
1935   }
1936 
1937   branch(false, false);
1938   __ bind(not_taken);
1939   __ profile_not_taken_branch(x10);
1940 }
1941 
1942 void TemplateTable::if_nullcmp(Condition cc)
1943 {
1944   transition(atos, vtos);
1945   // assume branch is more often taken than not (loops use backward branches)
1946   Label not_taken;
1947   if (cc == equal) {
1948     __ bnez(x10, not_taken);
1949   } else {
1950     __ beqz(x10, not_taken);
1951   }
1952   branch(false, false);
1953   __ bind(not_taken);
1954   __ profile_not_taken_branch(x10);
1955 }
1956 
1957 void TemplateTable::if_acmp(Condition cc)
1958 {
1959   transition(atos, vtos);
1960   // assume branch is more often taken than not (loops use backward branches)
1961   Label not_taken;
1962   __ pop_ptr(x11);
1963 
1964   if (cc == equal) {
1965     __ bne(x11, x10, not_taken);
1966   } else if (cc == not_equal) {
1967     __ beq(x11, x10, not_taken);
1968   }
1969   branch(false, false);
1970   __ bind(not_taken);
1971   __ profile_not_taken_branch(x10);
1972 }
1973 
1974 void TemplateTable::ret() {
1975   transition(vtos, vtos);
1976   // We might be moving to a safepoint.  The thread which calls
1977   // Interpreter::notice_safepoints() will effectively flush its cache
1978   // when it makes a system call, but we need to do something to
1979   // ensure that we see the changed dispatch table.
1980   __ membar(MacroAssembler::LoadLoad);
1981 
1982   locals_index(x11);
1983   __ ld(x11, aaddress(x11, t1, _masm)); // get return bci, compute return bcp
1984   __ profile_ret(x11, x12);
1985   __ ld(xbcp, Address(xmethod, Method::const_offset()));
1986   __ add(xbcp, xbcp, x11);
1987   __ addi(xbcp, xbcp, in_bytes(ConstMethod::codes_offset()));
1988   __ dispatch_next(vtos, 0, /*generate_poll*/true);
1989 }
1990 
1991 void TemplateTable::wide_ret() {
1992   transition(vtos, vtos);
1993   locals_index_wide(x11);
1994   __ ld(x11, aaddress(x11, t0, _masm)); // get return bci, compute return bcp
1995   __ profile_ret(x11, x12);
1996   __ ld(xbcp, Address(xmethod, Method::const_offset()));
1997   __ add(xbcp, xbcp, x11);
1998   __ add(xbcp, xbcp, in_bytes(ConstMethod::codes_offset()));
1999   __ dispatch_next(vtos, 0, /*generate_poll*/true);
2000 }
2001 
2002 void TemplateTable::tableswitch() {
2003   Label default_case, continue_execution;
2004   transition(itos, vtos);
2005   // align xbcp
2006   __ la(x11, at_bcp(BytesPerInt));
2007   __ andi(x11, x11, -BytesPerInt);
2008   // load lo & hi
2009   __ lwu(x12, Address(x11, BytesPerInt));
2010   __ lwu(x13, Address(x11, 2 * BytesPerInt));
2011   __ grevw(x12, x12); // reverse bytes in word (32bit) and sign-extend
2012   __ grevw(x13, x13); // reverse bytes in word (32bit) and sign-extend
2013   // check against lo & hi
2014   __ blt(x10, x12, default_case);
2015   __ bgt(x10, x13, default_case);
2016   // lookup dispatch offset
2017   __ subw(x10, x10, x12);
2018   __ slli(t0, x10, 2);
2019   __ add(x13, x11, t0);
2020   __ lwu(x13, Address(x13, 3 * BytesPerInt));
2021   __ profile_switch_case(x10, x11, x12);
2022   // continue execution
2023   __ bind(continue_execution);
2024   __ grevw(x13, x13); // reverse bytes in word (32bit) and sign-extend
2025   __ add(xbcp, xbcp, x13);
2026   __ load_unsigned_byte(t0, Address(xbcp));
2027   __ dispatch_only(vtos, /*generate_poll*/true);
2028   // handle default
2029   __ bind(default_case);
2030   __ profile_switch_default(x10);
2031   __ lwu(x13, Address(x11, 0));
2032   __ j(continue_execution);
2033 }
2034 
2035 void TemplateTable::lookupswitch() {
2036   transition(itos, itos);
2037   __ stop("lookupswitch bytecode should have been rewritten");
2038 }
2039 
2040 void TemplateTable::fast_linearswitch() {
2041   transition(itos, vtos);
2042   Label loop_entry, loop, found, continue_execution;
2043   // bswap x10 so we can avoid bswapping the table entries
2044   __ grevw(x10, x10); // reverse bytes in word (32bit) and sign-extend
2045   // align xbcp
2046   __ la(x9, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2047                                     // this instruction (change offsets
2048                                     // below)
2049   __ andi(x9, x9, -BytesPerInt);
2050   // set counter
2051   __ lwu(x11, Address(x9, BytesPerInt));
2052   __ grev32(x11, x11);
2053   __ j(loop_entry);
2054   // table search
2055   __ bind(loop);
2056   __ slli(t0, x11, 3);
2057   __ add(t0, x9, t0);
2058   __ lw(t0, Address(t0, 2 * BytesPerInt));
2059   __ beq(x10, t0, found);
2060   __ bind(loop_entry);
2061   __ addi(x11, x11, -1);
2062   __ bgez(x11, loop);
2063   // default case
2064   __ profile_switch_default(x10);
2065   __ lwu(x13, Address(x9, 0));
2066   __ j(continue_execution);
2067   // entry found -> get offset
2068   __ bind(found);
2069   __ slli(t0, x11, 3);
2070   __ add(t0, x9, t0);
2071   __ lwu(x13, Address(t0, 3 * BytesPerInt));
2072   __ profile_switch_case(x11, x10, x9);
2073   // continue execution
2074   __ bind(continue_execution);
2075   __ grevw(x13, x13); // reverse bytes in word (32bit) and sign-extend
2076   __ add(xbcp, xbcp, x13);
2077   __ lbu(t0, Address(xbcp, 0));
2078   __ dispatch_only(vtos, /*generate_poll*/true);
2079 }
2080 
2081 void TemplateTable::fast_binaryswitch() {
2082   transition(itos, vtos);
2083   // Implementation using the following core algorithm:
2084   //
2085   // int binary_search(int key, LookupswitchPair* array, int n)
2086   //   binary_search start:
2087   //   #Binary search according to "Methodik des Programmierens" by
2088   //   # Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2089   //   int i = 0;
2090   //   int j = n;
2091   //   while (i + 1 < j) do
2092   //     # invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2093   //     # with      Q: for all i: 0 <= i < n: key < a[i]
2094   //     # where a stands for the array and assuming that the (inexisting)
2095   //     # element a[n] is infinitely big.
2096   //     int h = (i + j) >> 1
2097   //     # i < h < j
2098   //     if (key < array[h].fast_match())
2099   //     then [j = h]
2100   //     else [i = h]
2101   //   end
2102   //   # R: a[i] <= key < a[i+1] or Q
2103   //   # (i.e., if key is within array, i is the correct index)
2104   //   return i
2105   // binary_search end
2106 
2107 
2108   // Register allocation
2109   const Register key   = x10; // already set (tosca)
2110   const Register array = x11;
2111   const Register i     = x12;
2112   const Register j     = x13;
2113   const Register h     = x14;
2114   const Register temp  = x15;
2115 
2116   // Find array start
2117   __ la(array, at_bcp(3 * BytesPerInt));  // btw: should be able to
2118                                           // get rid of this
2119                                           // instruction (change
2120                                           // offsets below)
2121   __ andi(array, array, -BytesPerInt);
2122 
2123   // Initialize i & j
2124   __ mv(i, zr);                            // i = 0
2125   __ lwu(j, Address(array, -BytesPerInt)); // j = length(array)
2126 
2127   // Convert j into native byteordering
2128   __ grev32(j, j);
2129 
2130   // And start
2131   Label entry;
2132   __ j(entry);
2133 
2134   // binary search loop
2135   {
2136     Label loop;
2137     __ bind(loop);
2138     __ addw(h, i, j);                           // h = i + j
2139     __ srliw(h, h, 1);                          // h = (i + j) >> 1
2140     // if [key < array[h].fast_match()]
2141     // then [j = h]
2142     // else [i = h]
2143     // Convert array[h].match to native byte-ordering before compare
2144     __ slli(temp, h, 3);
2145     __ add(temp, array, temp);
2146     __ ld(temp, Address(temp, 0));
2147     __ grevw(temp, temp); // reverse bytes in word (32bit) and sign-extend
2148 
2149     Label L_done, L_greater;
2150     __ bge(key, temp, L_greater);
2151     // if [key < array[h].fast_match()] then j = h
2152     __ mv(j, h);
2153     __ j(L_done);
2154     __ bind(L_greater);
2155     // if [key >= array[h].fast_match()] then i = h
2156     __ mv(i, h);
2157     __ bind(L_done);
2158 
2159     // while [i + 1 < j]
2160     __ bind(entry);
2161     __ addiw(h, i, 1);         // i + 1
2162     __ blt(h, j, loop);        // i + 1 < j
2163   }
2164 
2165   // end of binary search, result index is i (must check again!)
2166   Label default_case;
2167   // Convert array[i].match to native byte-ordering before compare
2168   __ slli(temp, i, 3);
2169   __ add(temp, array, temp);
2170   __ ld(temp, Address(temp, 0));
2171   __ grevw(temp, temp); // reverse bytes in word (32bit) and sign-extend
2172   __ bne(key, temp, default_case);
2173 
2174   // entry found -> j = offset
2175   __ slli(temp, i, 3);
2176   __ add(temp, array, temp);
2177   __ lwu(j, Address(temp, BytesPerInt));
2178   __ profile_switch_case(i, key, array);
2179   __ grevw(j, j); // reverse bytes in word (32bit) and sign-extend
2180 
2181   __ add(temp, xbcp, j);
2182   __ load_unsigned_byte(t0, Address(temp, 0));
2183 
2184   __ add(xbcp, xbcp, j);
2185   __ la(xbcp, Address(xbcp, 0));
2186   __ dispatch_only(vtos, /*generate_poll*/true);
2187 
2188   // default case -> j = default offset
2189   __ bind(default_case);
2190   __ profile_switch_default(i);
2191   __ lwu(j, Address(array, -2 * BytesPerInt));
2192   __ grevw(j, j); // reverse bytes in word (32bit) and sign-extend
2193 
2194   __ add(temp, xbcp, j);
2195   __ load_unsigned_byte(t0, Address(temp, 0));
2196 
2197   __ add(xbcp, xbcp, j);
2198   __ la(xbcp, Address(xbcp, 0));
2199   __ dispatch_only(vtos, /*generate_poll*/true);
2200 }
2201 
2202 void TemplateTable::_return(TosState state)
2203 {
2204   transition(state, state);
2205   assert(_desc->calls_vm(),
2206          "inconsistent calls_vm information"); // call in remove_activation
2207 
2208   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2209     assert(state == vtos, "only valid state");
2210 
2211     __ ld(c_rarg1, aaddress(0));
2212     __ load_klass(x13, c_rarg1);
2213     __ lwu(x13, Address(x13, Klass::access_flags_offset()));
2214     Label skip_register_finalizer;
2215     __ andi(t0, x13, JVM_ACC_HAS_FINALIZER);
2216     __ beqz(t0, skip_register_finalizer);
2217 
2218     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2219 
2220     __ bind(skip_register_finalizer);
2221   }
2222 
2223   // Issue a StoreStore barrier after all stores but before return
2224   // from any constructor for any class with a final field. We don't
2225   // know if this is a finalizer, so we always do so.
2226   if (_desc->bytecode() == Bytecodes::_return) {
2227     __ membar(MacroAssembler::StoreStore);
2228   }
2229 
2230   // Narrow result if state is itos but result type is smaller.
2231   // Need to narrow in the return bytecode rather than in generate_return_entry
2232   // since compiled code callers expect the result to already be narrowed.
2233   if (state == itos) {
2234     __ narrow(x10);
2235   }
2236 
2237   __ remove_activation(state);
2238   __ ret();
2239 }
2240 
2241 
2242 // ----------------------------------------------------------------------------
2243 // Volatile variables demand their effects be made known to all CPU's
2244 // in order.  Store buffers on most chips allow reads & writes to
2245 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2246 // without some kind of memory barrier (i.e., it's not sufficient that
2247 // the interpreter does not reorder volatile references, the hardware
2248 // also must not reorder them).
2249 //
2250 // According to the new Java Memory Model (JMM):
2251 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2252 //     writes act as aquire & release, so:
2253 // (2) A read cannot let unrelated NON-volatile memory refs that
2254 //     happen after the read float up to before the read.  It's OK for
2255 //     non-volatile memory refs that happen before the volatile read to
2256 //     float down below it.
2257 // (3) Similar a volatile write cannot let unrelated NON-volatile
2258 //     memory refs that happen BEFORE the write float down to after the
2259 //     write.  It's OK for non-volatile memory refs that happen after the
2260 //     volatile write to float up before it.
2261 //
2262 // We only put in barriers around volatile refs (they are expensive),
2263 // not _between_ memory refs (that would require us to track the
2264 // flavor of the previous memory refs).  Requirements (2) and (3)
2265 // require some barriers before volatile stores and after volatile
2266 // loads.  These nearly cover requirement (1) but miss the
2267 // volatile-store-volatile-load case.  This final case is placed after
2268 // volatile-stores although it could just as well go before
2269 // volatile-loads.
2270 
2271 void TemplateTable::resolve_cache_and_index(int byte_no,
2272                                             Register Rcache,
2273                                             Register index,
2274                                             size_t index_size) {
2275   const Register temp = x9;
2276   assert_different_registers(Rcache, index, temp);
2277 
2278   Label resolved, clinit_barrier_slow;
2279 
2280   Bytecodes::Code code = bytecode();
2281   switch (code) {
2282     case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2283     case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2284     default: break;
2285   }
2286 
2287   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2288   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2289   __ mv(t0, (int) code);
2290   __ beq(temp, t0, resolved);
2291 
2292   // resolve first time through
2293   // Class initialization barrier slow path lands here as well.
2294   __ bind(clinit_barrier_slow);
2295 
2296   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2297   __ mv(temp, (int) code);
2298   __ call_VM(noreg, entry, temp);
2299 
2300   // Update registers with resolved info
2301   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2302   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2303   // so all clients ofthis method must be modified accordingly
2304   __ bind(resolved);
2305 
2306   // Class initialization barrier for static methods
2307   if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
2308     __ load_resolved_method_at_index(byte_no, temp, Rcache);
2309     __ load_method_holder(temp, temp);
2310     __ clinit_barrier(temp, t0, NULL, &clinit_barrier_slow);
2311   }
2312 }
2313 
2314 // The Rcache and index registers must be set before call
2315 // n.b unlike x86 cache already includes the index offset
2316 void TemplateTable::load_field_cp_cache_entry(Register obj,
2317                                               Register cache,
2318                                               Register index,
2319                                               Register off,
2320                                               Register flags,
2321                                               bool is_static = false) {
2322   assert_different_registers(cache, index, flags, off);
2323 
2324   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2325   // Field offset
2326   __ ld(off, Address(cache, in_bytes(cp_base_offset +
2327                                      ConstantPoolCacheEntry::f2_offset())));
2328   // Flags
2329   __ lwu(flags, Address(cache, in_bytes(cp_base_offset +
2330                                         ConstantPoolCacheEntry::flags_offset())));
2331 
2332   // klass overwrite register
2333   if (is_static) {
2334     __ ld(obj, Address(cache, in_bytes(cp_base_offset +
2335                                        ConstantPoolCacheEntry::f1_offset())));
2336     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2337     __ ld(obj, Address(obj, mirror_offset));
2338     __ resolve_oop_handle(obj);
2339   }
2340 }
2341 
2342 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2343                                                Register method,
2344                                                Register itable_index,
2345                                                Register flags,
2346                                                bool is_invokevirtual,
2347                                                bool is_invokevfinal, /*unused*/
2348                                                bool is_invokedynamic) {
2349   // setup registers
2350   const Register cache = t1;
2351   const Register index = x14;
2352   assert_different_registers(method, flags);
2353   assert_different_registers(method, cache, index);
2354   assert_different_registers(itable_index, flags);
2355   assert_different_registers(itable_index, cache, index);
2356   // determine constant pool cache field offsets
2357   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2358   const int method_offset = in_bytes(ConstantPoolCache::base_offset() +
2359                                      (is_invokevirtual ?
2360                                       ConstantPoolCacheEntry::f2_offset() :
2361                                       ConstantPoolCacheEntry::f1_offset()));
2362   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2363                                     ConstantPoolCacheEntry::flags_offset());
2364   // access constant pool cache fields
2365   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2366                                     ConstantPoolCacheEntry::f2_offset());
2367 
2368   const size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2369   resolve_cache_and_index(byte_no, cache, index, index_size);
2370   __ ld(method, Address(cache, method_offset));
2371 
2372   if (itable_index != noreg) {
2373     __ ld(itable_index, Address(cache, index_offset));
2374   }
2375   __ lwu(flags, Address(cache, flags_offset));
2376 }
2377 
2378 // The registers cache and index expected to be set before call.
2379 // Correct values of the cache and index registers are preserved.
2380 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2381                                             bool is_static, bool has_tos) {
2382   // do the JVMTI work here to avoid disturbing the register state below
2383   // We use c_rarg registers here beacause we want to use the register used in
2384   // the call to the VM
2385   if (JvmtiExport::can_post_field_access()) {
2386     // Check to see if a field access watch has been set before we
2387     // take the time to call into the VM.
2388     Label L1;
2389     assert_different_registers(cache, index, x10);
2390     int32_t offset = 0;
2391     __ la_patchable(t0, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()), offset);
2392     __ lwu(x10, Address(t0, offset));
2393 
2394     __ beqz(x10, L1);
2395 
2396     __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2397     __ la(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2398 
2399     if (is_static) {
2400       __ mv(c_rarg1, zr); // NULL object reference
2401     } else {
2402       __ ld(c_rarg1, at_tos()); // get object pointer without popping it
2403       __ verify_oop(c_rarg1);
2404     }
2405     // c_rarg1: object pointer or NULL
2406     // c_rarg2: cache entry pointer
2407     // c_rarg3: jvalue object on the stack
2408     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2409                                        InterpreterRuntime::post_field_access),
2410                                        c_rarg1, c_rarg2, c_rarg3);
2411     __ get_cache_and_index_at_bcp(cache, index, 1);
2412     __ bind(L1);
2413   }
2414 }
2415 
2416 void TemplateTable::pop_and_check_object(Register r)
2417 {
2418   __ pop_ptr(r);
2419   __ null_check(r);  // for field access must check obj.
2420   __ verify_oop(r);
2421 }
2422 
2423 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc)
2424 {
2425   const Register cache     = x12;
2426   const Register index     = x13;
2427   const Register obj       = x14;
2428   const Register off       = x9;
2429   const Register flags     = x10;
2430   const Register raw_flags = x16;
2431   const Register bc        = x14; // uses same reg as obj, so don't mix them
2432 
2433   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2434   jvmti_post_field_access(cache, index, is_static, false);
2435   load_field_cp_cache_entry(obj, cache, index, off, raw_flags, is_static);
2436 
2437   if (!is_static) {
2438     // obj is on the stack
2439     pop_and_check_object(obj);
2440   }
2441 
2442   __ add(off, obj, off);
2443   const Address field(off);
2444 
2445   Label Done, notByte, notBool, notInt, notShort, notChar,
2446               notLong, notFloat, notObj, notDouble;
2447 
2448   __ slli(flags, raw_flags, registerSize - (ConstantPoolCacheEntry::tos_state_shift +
2449                                             ConstantPoolCacheEntry::tos_state_bits));
2450   __ srli(flags, flags, registerSize - ConstantPoolCacheEntry::tos_state_bits);
2451 
2452   assert(btos == 0, "change code, btos != 0");
2453   __ bnez(flags, notByte);
2454 
2455   // Dont't rewrite getstatic, only getfield
2456   if (is_static) {
2457     rc = may_not_rewrite;
2458   }
2459 
2460   // btos
2461   __ access_load_at(T_BYTE, IN_HEAP, x10, field, noreg, noreg);
2462   __ push(btos);
2463   // Rewrite bytecode to be faster
2464   if (rc == may_rewrite) {
2465     patch_bytecode(Bytecodes::_fast_bgetfield, bc, x11);
2466   }
2467   __ j(Done);
2468 
2469   __ bind(notByte);
2470   __ sub(t0, flags, (u1)ztos);
2471   __ bnez(t0, notBool);
2472 
2473   // ztos (same code as btos)
2474   __ access_load_at(T_BOOLEAN, IN_HEAP, x10, field, noreg, noreg);
2475   __ push(ztos);
2476   // Rewirte bytecode to be faster
2477   if (rc == may_rewrite) {
2478     // uses btos rewriting, no truncating to t/f bit is needed for getfield
2479     patch_bytecode(Bytecodes::_fast_bgetfield, bc, x11);
2480   }
2481   __ j(Done);
2482 
2483   __ bind(notBool);
2484   __ sub(t0, flags, (u1)atos);
2485   __ bnez(t0, notObj);
2486   // atos
2487   do_oop_load(_masm, field, x10, IN_HEAP);
2488   __ push(atos);
2489   if (rc == may_rewrite) {
2490     patch_bytecode(Bytecodes::_fast_agetfield, bc, x11);
2491   }
2492   __ j(Done);
2493 
2494   __ bind(notObj);
2495   __ sub(t0, flags, (u1)itos);
2496   __ bnez(t0, notInt);
2497   // itos
2498   __ access_load_at(T_INT, IN_HEAP, x10, field, noreg, noreg);
2499   __ addw(x10, x10, zr); // signed extended
2500   __ push(itos);
2501   // Rewrite bytecode to be faster
2502   if (rc == may_rewrite) {
2503     patch_bytecode(Bytecodes::_fast_igetfield, bc, x11);
2504   }
2505   __ j(Done);
2506 
2507   __ bind(notInt);
2508   __ sub(t0, flags, (u1)ctos);
2509   __ bnez(t0, notChar);
2510   // ctos
2511   __ access_load_at(T_CHAR, IN_HEAP, x10, field, noreg, noreg);
2512   __ push(ctos);
2513   // Rewrite bytecode to be faster
2514   if (rc == may_rewrite) {
2515     patch_bytecode(Bytecodes::_fast_cgetfield, bc, x11);
2516   }
2517   __ j(Done);
2518 
2519   __ bind(notChar);
2520   __ sub(t0, flags, (u1)stos);
2521   __ bnez(t0, notShort);
2522   // stos
2523   __ access_load_at(T_SHORT, IN_HEAP, x10, field, noreg, noreg);
2524   __ push(stos);
2525   // Rewrite bytecode to be faster
2526   if (rc == may_rewrite) {
2527     patch_bytecode(Bytecodes::_fast_sgetfield, bc, x11);
2528   }
2529   __ j(Done);
2530 
2531   __ bind(notShort);
2532   __ sub(t0, flags, (u1)ltos);
2533   __ bnez(t0, notLong);
2534   // ltos
2535   __ access_load_at(T_LONG, IN_HEAP, x10, field, noreg, noreg);
2536   __ push(ltos);
2537   // Rewrite bytecode to be faster
2538   if (rc == may_rewrite) {
2539     patch_bytecode(Bytecodes::_fast_lgetfield, bc, x11);
2540   }
2541   __ j(Done);
2542 
2543   __ bind(notLong);
2544   __ sub(t0, flags, (u1)ftos);
2545   __ bnez(t0, notFloat);
2546   // ftos
2547   __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2548   __ push(ftos);
2549   // Rewrite bytecode to be faster
2550   if (rc == may_rewrite) {
2551     patch_bytecode(Bytecodes::_fast_fgetfield, bc, x11);
2552   }
2553   __ j(Done);
2554 
2555   __ bind(notFloat);
2556 #ifdef ASSERT
2557   __ sub(t0, flags, (u1)dtos);
2558   __ bnez(t0, notDouble);
2559 #endif
2560   // dtos
2561   __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2562   __ push(dtos);
2563   // Rewrite bytecode to be faster
2564   if (rc == may_rewrite) {
2565     patch_bytecode(Bytecodes::_fast_dgetfield, bc, x11);
2566   }
2567 #ifdef ASSERT
2568   __ j(Done);
2569 
2570   __ bind(notDouble);
2571   __ stop("Bad state");
2572 #endif
2573 
2574   __ bind(Done);
2575 
2576   Label notVolatile;
2577   __ andi(t0, raw_flags, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
2578   __ beqz(t0, notVolatile);
2579   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2580   __ bind(notVolatile);
2581 }
2582 
2583 void TemplateTable::getfield(int byte_no)
2584 {
2585   getfield_or_static(byte_no, false);
2586 }
2587 
2588 void TemplateTable::nofast_getfield(int byte_no) {
2589   getfield_or_static(byte_no, false, may_not_rewrite);
2590 }
2591 
2592 void TemplateTable::getstatic(int byte_no)
2593 {
2594   getfield_or_static(byte_no, true);
2595 }
2596 
2597 // The registers cache and index expected to be set before call.
2598 // The function may destroy various registers, just not the cache and index registers.
2599 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2600   transition(vtos, vtos);
2601 
2602   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2603 
2604   if (JvmtiExport::can_post_field_modification()) {
2605     // Check to see if a field modification watch has been set before
2606     // we take the time to call into the VM.
2607     Label L1;
2608     assert_different_registers(cache, index, x10);
2609     int32_t offset = 0;
2610     __ la_patchable(t0, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()), offset);
2611     __ lwu(x10, Address(t0, offset));
2612     __ beqz(x10, L1);
2613 
2614     __ get_cache_and_index_at_bcp(c_rarg2, t0, 1);
2615 
2616     if (is_static) {
2617       // Life is simple. Null out the object pointer.
2618       __ mv(c_rarg1, zr);
2619     } else {
2620       // Life is harder. The stack holds the value on top, followed by
2621       // the object. We don't know the size of the value, though; it
2622       // could be one or two words depending on its type. As a result,
2623       // we must find the type to determine where the object is.
2624       __ lwu(c_rarg3, Address(c_rarg2,
2625                               in_bytes(cp_base_offset +
2626                                        ConstantPoolCacheEntry::flags_offset())));
2627       __ srli(c_rarg3, c_rarg3, ConstantPoolCacheEntry::tos_state_shift);
2628       ConstantPoolCacheEntry::verify_tos_state_shift();
2629       Label nope2, done, ok;
2630       __ ld(c_rarg1, at_tos_p1());   // initially assume a one word jvalue
2631       __ sub(t0, c_rarg3, ltos);
2632       __ beqz(t0, ok);
2633       __ sub(t0, c_rarg3, dtos);
2634       __ bnez(t0, nope2);
2635       __ bind(ok);
2636       __ ld(c_rarg1, at_tos_p2());  // ltos (two word jvalue);
2637       __ bind(nope2);
2638     }
2639     // cache entry pointer
2640     __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2641     // object (tos)
2642     __ mv(c_rarg3, esp);
2643     // c_rarg1: object pointer set up above (NULL if static)
2644     // c_rarg2: cache entry pointer
2645     // c_rarg3: jvalue object on  the stack
2646     __ call_VM(noreg,
2647                CAST_FROM_FN_PTR(address,
2648                                 InterpreterRuntime::post_field_modification),
2649                                 c_rarg1, c_rarg2, c_rarg3);
2650     __ get_cache_and_index_at_bcp(cache, index, 1);
2651     __ bind(L1);
2652   }
2653 }
2654 
2655 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2656   transition(vtos, vtos);
2657 
2658   const Register cache = x12;
2659   const Register index = x13;
2660   const Register obj   = x12;
2661   const Register off   = x9;
2662   const Register flags = x10;
2663   const Register bc    = x14;
2664 
2665   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2666   jvmti_post_field_mod(cache, index, is_static);
2667   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2668 
2669   Label Done;
2670   __ mv(x15, flags);
2671 
2672   {
2673     Label notVolatile;
2674     __ andi(t0, x15, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
2675     __ beqz(t0, notVolatile);
2676     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2677     __ bind(notVolatile);
2678   }
2679 
2680   Label notByte, notBool, notInt, notShort, notChar,
2681         notLong, notFloat, notObj, notDouble;
2682 
2683   __ slli(flags, flags, registerSize - (ConstantPoolCacheEntry::tos_state_shift +
2684                                         ConstantPoolCacheEntry::tos_state_bits));
2685   __ srli(flags, flags, registerSize - ConstantPoolCacheEntry::tos_state_bits);
2686 
2687   assert(btos == 0, "change code, btos != 0");
2688   __ bnez(flags, notByte);
2689 
2690   // Don't rewrite putstatic, only putfield
2691   if (is_static) {
2692     rc = may_not_rewrite;
2693   }
2694 
2695   // btos
2696   {
2697     __ pop(btos);
2698     // field address
2699     if (!is_static) {
2700       pop_and_check_object(obj);
2701     }
2702     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2703     const Address field(off, 0); // off register as temparator register.
2704     __ access_store_at(T_BYTE, IN_HEAP, field, x10, noreg, noreg);
2705     if (rc == may_rewrite) {
2706       patch_bytecode(Bytecodes::_fast_bputfield, bc, x11, true, byte_no);
2707     }
2708     __ j(Done);
2709   }
2710 
2711   __ bind(notByte);
2712   __ sub(t0, flags, (u1)ztos);
2713   __ bnez(t0, notBool);
2714 
2715   // ztos
2716   {
2717     __ pop(ztos);
2718     // field address
2719     if (!is_static) {
2720       pop_and_check_object(obj);
2721     }
2722     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2723     const Address field(off, 0);
2724     __ access_store_at(T_BOOLEAN, IN_HEAP, field, x10, noreg, noreg);
2725     if (rc == may_rewrite) {
2726       patch_bytecode(Bytecodes::_fast_zputfield, bc, x11, true, byte_no);
2727     }
2728     __ j(Done);
2729   }
2730 
2731   __ bind(notBool);
2732   __ sub(t0, flags, (u1)atos);
2733   __ bnez(t0, notObj);
2734 
2735   // atos
2736   {
2737     __ pop(atos);
2738     // field address
2739     if (!is_static) {
2740       pop_and_check_object(obj);
2741     }
2742     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2743     const Address field(off, 0);
2744     // Store into the field
2745     do_oop_store(_masm, field, x10, IN_HEAP);
2746     if (rc == may_rewrite) {
2747       patch_bytecode(Bytecodes::_fast_aputfield, bc, x11, true, byte_no);
2748     }
2749     __ j(Done);
2750   }
2751 
2752   __ bind(notObj);
2753   __ sub(t0, flags, (u1)itos);
2754   __ bnez(t0, notInt);
2755 
2756   // itos
2757   {
2758     __ pop(itos);
2759     // field address
2760     if (!is_static) {
2761       pop_and_check_object(obj);
2762     }
2763     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2764     const Address field(off, 0);
2765     __ access_store_at(T_INT, IN_HEAP, field, x10, noreg, noreg);
2766     if (rc == may_rewrite) {
2767       patch_bytecode(Bytecodes::_fast_iputfield, bc, x11, true, byte_no);
2768     }
2769     __ j(Done);
2770   }
2771 
2772   __ bind(notInt);
2773   __ sub(t0, flags, (u1)ctos);
2774   __ bnez(t0, notChar);
2775 
2776   // ctos
2777   {
2778     __ pop(ctos);
2779     // field address
2780     if (!is_static) {
2781       pop_and_check_object(obj);
2782     }
2783     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2784     const Address field(off, 0);
2785     __ access_store_at(T_CHAR, IN_HEAP, field, x10, noreg, noreg);
2786     if (rc == may_rewrite) {
2787       patch_bytecode(Bytecodes::_fast_cputfield, bc, x11, true, byte_no);
2788     }
2789     __ j(Done);
2790   }
2791 
2792   __ bind(notChar);
2793   __ sub(t0, flags, (u1)stos);
2794   __ bnez(t0, notShort);
2795 
2796   // stos
2797   {
2798     __ pop(stos);
2799     // field address
2800     if (!is_static) {
2801       pop_and_check_object(obj);
2802     }
2803     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2804     const Address field(off, 0);
2805     __ access_store_at(T_SHORT, IN_HEAP, field, x10, noreg, noreg);
2806     if (rc == may_rewrite) {
2807       patch_bytecode(Bytecodes::_fast_sputfield, bc, x11, true, byte_no);
2808     }
2809     __ j(Done);
2810   }
2811 
2812   __ bind(notShort);
2813   __ sub(t0, flags, (u1)ltos);
2814   __ bnez(t0, notLong);
2815 
2816   // ltos
2817   {
2818     __ pop(ltos);
2819     // field address
2820     if (!is_static) {
2821       pop_and_check_object(obj);
2822     }
2823     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2824     const Address field(off, 0);
2825     __ access_store_at(T_LONG, IN_HEAP, field, x10, noreg, noreg);
2826     if (rc == may_rewrite) {
2827       patch_bytecode(Bytecodes::_fast_lputfield, bc, x11, true, byte_no);
2828     }
2829     __ j(Done);
2830   }
2831 
2832   __ bind(notLong);
2833   __ sub(t0, flags, (u1)ftos);
2834   __ bnez(t0, notFloat);
2835 
2836   // ftos
2837   {
2838     __ pop(ftos);
2839     // field address
2840     if (!is_static) {
2841       pop_and_check_object(obj);
2842     }
2843     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2844     const Address field(off, 0);
2845     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg);
2846     if (rc == may_rewrite) {
2847       patch_bytecode(Bytecodes::_fast_fputfield, bc, x11, true, byte_no);
2848     }
2849     __ j(Done);
2850   }
2851 
2852   __ bind(notFloat);
2853 #ifdef ASSERT
2854   __ sub(t0, flags, (u1)dtos);
2855   __ bnez(t0, notDouble);
2856 #endif
2857 
2858   // dtos
2859   {
2860     __ pop(dtos);
2861     // field address
2862     if (!is_static) {
2863       pop_and_check_object(obj);
2864     }
2865     __ add(off, obj, off); // if static, obj from cache, else obj from stack.
2866     const Address field(off, 0);
2867     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg);
2868     if (rc == may_rewrite) {
2869       patch_bytecode(Bytecodes::_fast_dputfield, bc, x11, true, byte_no);
2870     }
2871   }
2872 
2873 #ifdef ASSERT
2874   __ j(Done);
2875 
2876   __ bind(notDouble);
2877   __ stop("Bad state");
2878 #endif
2879 
2880   __ bind(Done);
2881 
2882   {
2883     Label notVolatile;
2884     __ andi(t0, x15, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
2885     __ beqz(t0, notVolatile);
2886     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
2887     __ bind(notVolatile);
2888   }
2889 }
2890 
2891 void TemplateTable::putfield(int byte_no)
2892 {
2893   putfield_or_static(byte_no, false);
2894 }
2895 
2896 void TemplateTable::nofast_putfield(int byte_no) {
2897   putfield_or_static(byte_no, false, may_not_rewrite);
2898 }
2899 
2900 void TemplateTable::putstatic(int byte_no) {
2901   putfield_or_static(byte_no, true);
2902 }
2903 
2904 void TemplateTable::jvmti_post_fast_field_mod()
2905 {
2906   if (JvmtiExport::can_post_field_modification()) {
2907     // Check to see if a field modification watch has been set before
2908     // we take the time to call into the VM.
2909     Label L2;
2910     int32_t offset = 0;
2911     __ la_patchable(t0, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()), offset);
2912     __ lwu(c_rarg3, Address(t0, offset));
2913     __ beqz(c_rarg3, L2);
2914     __ pop_ptr(x9);                  // copy the object pointer from tos
2915     __ verify_oop(x9);
2916     __ push_ptr(x9);                 // put the object pointer back on tos
2917     // Save tos values before call_VM() clobbers them. Since we have
2918     // to do it for every data type, we use the saved values as the
2919     // jvalue object.
2920     switch (bytecode()) {          // load values into the jvalue object
2921       case Bytecodes::_fast_aputfield: __ push_ptr(x10); break;
2922       case Bytecodes::_fast_bputfield: // fall through
2923       case Bytecodes::_fast_zputfield: // fall through
2924       case Bytecodes::_fast_sputfield: // fall through
2925       case Bytecodes::_fast_cputfield: // fall through
2926       case Bytecodes::_fast_iputfield: __ push_i(x10); break;
2927       case Bytecodes::_fast_dputfield: __ push_d(); break;
2928       case Bytecodes::_fast_fputfield: __ push_f(); break;
2929       case Bytecodes::_fast_lputfield: __ push_l(x10); break;
2930 
2931       default:
2932         ShouldNotReachHere();
2933     }
2934     __ mv(c_rarg3, esp);             // points to jvalue on the stack
2935     // access constant pool cache entry
2936     __ get_cache_entry_pointer_at_bcp(c_rarg2, x10, 1);
2937     __ verify_oop(x9);
2938     // x9: object pointer copied above
2939     // c_rarg2: cache entry pointer
2940     // c_rarg3: jvalue object on the stack
2941     __ call_VM(noreg,
2942                CAST_FROM_FN_PTR(address,
2943                                 InterpreterRuntime::post_field_modification),
2944                x9, c_rarg2, c_rarg3);
2945 
2946     switch (bytecode()) {             // restore tos values
2947       case Bytecodes::_fast_aputfield: __ pop_ptr(x10); break;
2948       case Bytecodes::_fast_bputfield: // fall through
2949       case Bytecodes::_fast_zputfield: // fall through
2950       case Bytecodes::_fast_sputfield: // fall through
2951       case Bytecodes::_fast_cputfield: // fall through
2952       case Bytecodes::_fast_iputfield: __ pop_i(x10); break;
2953       case Bytecodes::_fast_dputfield: __ pop_d(); break;
2954       case Bytecodes::_fast_fputfield: __ pop_f(); break;
2955       case Bytecodes::_fast_lputfield: __ pop_l(x10); break;
2956       default: break;
2957     }
2958     __ bind(L2);
2959   }
2960 }
2961 
2962 void TemplateTable::fast_storefield(TosState state)
2963 {
2964   transition(state, vtos);
2965 
2966   ByteSize base = ConstantPoolCache::base_offset();
2967 
2968   jvmti_post_fast_field_mod();
2969 
2970   // access constant pool cache
2971   __ get_cache_and_index_at_bcp(x12, x11, 1);
2972 
2973   // Must prevent reordering of the following cp cache loads with bytecode load
2974   __ membar(MacroAssembler::LoadLoad);
2975 
2976   // test for volatile with x13
2977   __ lwu(x13, Address(x12, in_bytes(base +
2978                                     ConstantPoolCacheEntry::flags_offset())));
2979 
2980   // replace index with field offset from cache entry
2981   __ ld(x11, Address(x12, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
2982 
2983   {
2984     Label notVolatile;
2985     __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
2986     __ beqz(t0, notVolatile);
2987     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2988     __ bind(notVolatile);
2989   }
2990 
2991   // Get object from stack
2992   pop_and_check_object(x12);
2993 
2994   // field address
2995   __ add(x11, x12, x11);
2996   const Address field(x11, 0);
2997 
2998   // access field
2999   switch (bytecode()) {
3000     case Bytecodes::_fast_aputfield:
3001       do_oop_store(_masm, field, x10, IN_HEAP);
3002       break;
3003     case Bytecodes::_fast_lputfield:
3004       __ access_store_at(T_LONG, IN_HEAP, field, x10, noreg, noreg);
3005       break;
3006     case Bytecodes::_fast_iputfield:
3007       __ access_store_at(T_INT, IN_HEAP, field, x10, noreg, noreg);
3008       break;
3009     case Bytecodes::_fast_zputfield:
3010       __ access_store_at(T_BOOLEAN, IN_HEAP, field, x10, noreg, noreg);
3011       break;
3012     case Bytecodes::_fast_bputfield:
3013       __ access_store_at(T_BYTE, IN_HEAP, field, x10, noreg, noreg);
3014       break;
3015     case Bytecodes::_fast_sputfield:
3016       __ access_store_at(T_SHORT, IN_HEAP, field, x10, noreg, noreg);
3017       break;
3018     case Bytecodes::_fast_cputfield:
3019       __ access_store_at(T_CHAR, IN_HEAP, field, x10, noreg, noreg);
3020       break;
3021     case Bytecodes::_fast_fputfield:
3022       __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg);
3023       break;
3024     case Bytecodes::_fast_dputfield:
3025       __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg);
3026       break;
3027     default:
3028       ShouldNotReachHere();
3029   }
3030 
3031   {
3032     Label notVolatile;
3033     __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
3034     __ beqz(t0, notVolatile);
3035     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
3036     __ bind(notVolatile);
3037   }
3038 }
3039 
3040 void TemplateTable::fast_accessfield(TosState state)
3041 {
3042   transition(atos, state);
3043   // Do the JVMTI work here to avoid disturbing the register state below
3044   if (JvmtiExport::can_post_field_access()) {
3045     // Check to see if a field access watch has been set before we
3046     // take the time to call into the VM.
3047     Label L1;
3048     int32_t offset = 0;
3049     __ la_patchable(t0, ExternalAddress((address)JvmtiExport::get_field_access_count_addr()), offset);
3050     __ lwu(x12, Address(t0, offset));
3051     __ beqz(x12, L1);
3052     // access constant pool cache entry
3053     __ get_cache_entry_pointer_at_bcp(c_rarg2, t1, 1);
3054     __ verify_oop(x10);
3055     __ push_ptr(x10);  // save object pointer before call_VM() clobbers it
3056     __ mv(c_rarg1, x10);
3057     // c_rarg1: object pointer copied above
3058     // c_rarg2: cache entry pointer
3059     __ call_VM(noreg,
3060                CAST_FROM_FN_PTR(address,
3061                                 InterpreterRuntime::post_field_access),
3062                c_rarg1, c_rarg2);
3063     __ pop_ptr(x10); // restore object pointer
3064     __ bind(L1);
3065   }
3066 
3067   // access constant pool cache
3068   __ get_cache_and_index_at_bcp(x12, x11, 1);
3069 
3070   // Must prevent reordering of the following cp cache loads with bytecode load
3071   __ membar(MacroAssembler::LoadLoad);
3072 
3073   __ ld(x11, Address(x12, in_bytes(ConstantPoolCache::base_offset() +
3074                                    ConstantPoolCacheEntry::f2_offset())));
3075   __ lwu(x13, Address(x12, in_bytes(ConstantPoolCache::base_offset() +
3076                                     ConstantPoolCacheEntry::flags_offset())));
3077 
3078   // x10: object
3079   __ verify_oop(x10);
3080   __ null_check(x10);
3081   __ add(x11, x10, x11);
3082   const Address field(x11, 0);
3083 
3084   // access field
3085   switch (bytecode()) {
3086     case Bytecodes::_fast_agetfield:
3087       do_oop_load(_masm, field, x10, IN_HEAP);
3088       __ verify_oop(x10);
3089       break;
3090     case Bytecodes::_fast_lgetfield:
3091       __ access_load_at(T_LONG, IN_HEAP, x10, field, noreg, noreg);
3092       break;
3093     case Bytecodes::_fast_igetfield:
3094       __ access_load_at(T_INT, IN_HEAP, x10, field, noreg, noreg);
3095       __ addw(x10, x10, zr); // signed extended
3096       break;
3097     case Bytecodes::_fast_bgetfield:
3098       __ access_load_at(T_BYTE, IN_HEAP, x10, field, noreg, noreg);
3099       break;
3100     case Bytecodes::_fast_sgetfield:
3101       __ access_load_at(T_SHORT, IN_HEAP, x10, field, noreg, noreg);
3102       break;
3103     case Bytecodes::_fast_cgetfield:
3104       __ access_load_at(T_CHAR, IN_HEAP, x10, field, noreg, noreg);
3105       break;
3106     case Bytecodes::_fast_fgetfield:
3107       __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
3108       break;
3109     case Bytecodes::_fast_dgetfield:
3110       __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* dtos */, field, noreg, noreg);
3111       break;
3112     default:
3113       ShouldNotReachHere();
3114   }
3115   {
3116     Label notVolatile;
3117     __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
3118     __ beqz(t0, notVolatile);
3119     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3120     __ bind(notVolatile);
3121   }
3122 }
3123 
3124 void TemplateTable::fast_xaccess(TosState state)
3125 {
3126   transition(vtos, state);
3127 
3128   // get receiver
3129   __ ld(x10, aaddress(0));
3130   // access constant pool cache
3131   __ get_cache_and_index_at_bcp(x12, x13, 2);
3132   __ ld(x11, Address(x12, in_bytes(ConstantPoolCache::base_offset() +
3133                                    ConstantPoolCacheEntry::f2_offset())));
3134 
3135   // make sure exception is reported in correct bcp range (getfield is
3136   // next instruction)
3137   __ addi(xbcp, xbcp, 1);
3138   __ null_check(x10);
3139   switch (state) {
3140     case itos:
3141       __ add(x10, x10, x11);
3142       __ access_load_at(T_INT, IN_HEAP, x10, Address(x10, 0), noreg, noreg);
3143       __ addw(x10, x10, zr); // signed extended
3144       break;
3145     case atos:
3146       __ add(x10, x10, x11);
3147       do_oop_load(_masm, Address(x10, 0), x10, IN_HEAP);
3148       __ verify_oop(x10);
3149       break;
3150     case ftos:
3151       __ add(x10, x10, x11);
3152       __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, Address(x10), noreg, noreg);
3153       break;
3154     default:
3155       ShouldNotReachHere();
3156   }
3157 
3158   {
3159     Label notVolatile;
3160     __ lwu(x13, Address(x12, in_bytes(ConstantPoolCache::base_offset() +
3161                                       ConstantPoolCacheEntry::flags_offset())));
3162     __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_volatile_shift);
3163     __ beqz(t0, notVolatile);
3164     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3165     __ bind(notVolatile);
3166   }
3167 
3168   __ sub(xbcp, xbcp, 1);
3169 }
3170 
3171 //-----------------------------------------------------------------------------
3172 // Calls
3173 
3174 void TemplateTable::prepare_invoke(int byte_no,
3175                                    Register method, // linked method (or i-klass)
3176                                    Register index,  // itable index, MethodType, etc.
3177                                    Register recv,   // if caller wants to see it
3178                                    Register flags   // if caller wants to test it
3179                                    ) {
3180   // determine flags
3181   const Bytecodes::Code code = bytecode();
3182   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3183   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3184   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3185   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3186   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3187   const bool load_receiver       = (recv  != noreg);
3188   const bool save_flags          = (flags != noreg);
3189   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3190   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3191   assert(flags == noreg || flags == x13, "");
3192   assert(recv  == noreg || recv  == x12, "");
3193 
3194   // setup registers & access constant pool cache
3195   if (recv  == noreg) {
3196     recv  = x12;
3197   }
3198   if (flags == noreg) {
3199     flags = x13;
3200   }
3201   assert_different_registers(method, index, recv, flags);
3202 
3203   // save 'interpreter return address'
3204   __ save_bcp();
3205 
3206   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3207 
3208   // maybe push appendix to arguments (just before return address)
3209   if (is_invokedynamic || is_invokehandle) {
3210     Label L_no_push;
3211     __ andi(t0, flags, 1UL << ConstantPoolCacheEntry::has_appendix_shift);
3212     __ beqz(t0, L_no_push);
3213     // Push the appendix as a trailing parameter.
3214     // This must be done before we get the receiver,
3215     // since the parameter_size includes it.
3216     __ push_reg(x9);
3217     __ mv(x9, index);
3218     __ load_resolved_reference_at_index(index, x9);
3219     __ pop_reg(x9);
3220     __ push_reg(index);  // push appendix (MethodType, CallSite, etc.)
3221     __ bind(L_no_push);
3222   }
3223 
3224   // load receiver if needed (note: no return address pushed yet)
3225   if (load_receiver) {
3226     __ andi(recv, flags, ConstantPoolCacheEntry::parameter_size_mask); // parameter_size_mask = 1 << 8
3227     __ slli(t0, recv, 3);
3228     __ add(t0, esp, t0);
3229     __ ld(recv, Address(t0, -Interpreter::expr_offset_in_bytes(1)));
3230     __ verify_oop(recv);
3231   }
3232 
3233   // compute return type
3234   __ slli(t1, flags, registerSize - (ConstantPoolCacheEntry::tos_state_shift + ConstantPoolCacheEntry::tos_state_bits));
3235   __ srli(t1, t1, registerSize - ConstantPoolCacheEntry::tos_state_bits); // (1 << 5) - 4 --> 28~31==> t1:0~3
3236 
3237   // load return address
3238   {
3239     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3240     __ mv(t0, table_addr);
3241     __ slli(t1, t1, 3);
3242     __ add(t0, t0, t1);
3243     __ ld(lr, Address(t0, 0));
3244   }
3245 }
3246 
3247 void TemplateTable::invokevirtual_helper(Register index,
3248                                          Register recv,
3249                                          Register flags)
3250 {
3251   // Uses temporary registers x10, x13
3252   assert_different_registers(index, recv, x10, x13);
3253   // Test for an invoke of a final method
3254   Label notFinal;
3255   __ andi(t0, flags, 1UL << ConstantPoolCacheEntry::is_vfinal_shift);
3256   __ beqz(t0, notFinal);
3257 
3258   const Register method = index;  // method must be xmethod
3259   assert(method == xmethod, "Method must be xmethod for interpreter calling convention");
3260 
3261   // do the call - the index is actually the method to call
3262   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3263 
3264   // It's final, need a null check here!
3265   __ null_check(recv);
3266 
3267   // profile this call
3268   __ profile_final_call(x10);
3269   __ profile_arguments_type(x10, method, x14, true);
3270 
3271   __ jump_from_interpreted(method);
3272 
3273   __ bind(notFinal);
3274 
3275   // get receiver klass
3276   __ null_check(recv, oopDesc::klass_offset_in_bytes());
3277   __ load_klass(x10, recv);
3278 
3279   // profile this call
3280   __ profile_virtual_call(x10, xlocals, x13);
3281 
3282   // get target Method & entry point
3283   __ lookup_virtual_method(x10, index, method);
3284   __ profile_arguments_type(x13, method, x14, true);
3285   __ jump_from_interpreted(method);
3286 }
3287 
3288 void TemplateTable::invokevirtual(int byte_no)
3289 {
3290   transition(vtos, vtos);
3291   assert(byte_no == f2_byte, "use this argument");
3292 
3293   prepare_invoke(byte_no, xmethod, noreg, x12, x13);
3294 
3295   // xmethod: index (actually a Method*)
3296   // x12: receiver
3297   // x13: flags
3298 
3299   invokevirtual_helper(xmethod, x12, x13);
3300 }
3301 
3302 void TemplateTable::invokespecial(int byte_no)
3303 {
3304   transition(vtos, vtos);
3305   assert(byte_no == f1_byte, "use this argument");
3306 
3307   prepare_invoke(byte_no, xmethod, noreg,  // get f1 Method*
3308                  x12);  // get receiver also for null check
3309   __ verify_oop(x12);
3310   __ null_check(x12);
3311   // do the call
3312   __ profile_call(x10);
3313   __ profile_arguments_type(x10, xmethod, xbcp, false);
3314   __ jump_from_interpreted(xmethod);
3315 }
3316 
3317 void TemplateTable::invokestatic(int byte_no)
3318 {
3319   transition(vtos, vtos);
3320   assert(byte_no == f1_byte, "use this arugment");
3321 
3322   prepare_invoke(byte_no, xmethod);  // get f1 Method*
3323   // do the call
3324   __ profile_call(x10);
3325   __ profile_arguments_type(x10, xmethod, x14, false);
3326   __ jump_from_interpreted(xmethod);
3327 }
3328 
3329 void TemplateTable::fast_invokevfinal(int byte_no)
3330 {
3331   __ call_Unimplemented();
3332 }
3333 
3334 void TemplateTable::invokeinterface(int byte_no) {
3335   transition(vtos, vtos);
3336   assert(byte_no == f1_byte, "use this argument");
3337 
3338   prepare_invoke(byte_no, x10, xmethod,  // get f1 Klass*, f2 Method*
3339                  x12, x13);  // recv, flags
3340 
3341   // x10: interface klass (from f1)
3342   // xmethod: method (from f2)
3343   // x12: receiver
3344   // x13: flags
3345 
3346   // First check for Object case, then private interface method,
3347   // then regular interface method.
3348 
3349   // Special case of invokeinterface called for virtual method of
3350   // java.lang.Object. See cpCache.cpp for details
3351   Label notObjectMethod;
3352   __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_forced_virtual_shift);
3353   __ beqz(t0, notObjectMethod);
3354 
3355   invokevirtual_helper(xmethod, x12, x13);
3356   __ bind(notObjectMethod);
3357 
3358   Label no_such_interface;
3359 
3360   // Check for private method invocation - indicated by vfinal
3361   Label notVFinal;
3362   __ andi(t0, x13, 1UL << ConstantPoolCacheEntry::is_vfinal_shift);
3363   __ beqz(t0, notVFinal);
3364 
3365   // Check receiver klass into x13 - also a null check
3366   __ null_check(x12, oopDesc::klass_offset_in_bytes());
3367   __ load_klass(x13, x12);
3368 
3369   Label subtype;
3370   __ check_klass_subtype(x13, x10, x14, subtype);
3371   // If we get here the typecheck failed
3372   __ j(no_such_interface);
3373   __ bind(subtype);
3374 
3375   __ profile_final_call(x10);
3376   __ profile_arguments_type(x10, xmethod, x14, true);
3377   __ jump_from_interpreted(xmethod);
3378 
3379   __ bind(notVFinal);
3380 
3381   // Get receiver klass into x13 - also a null check
3382   __ restore_locals();
3383   __ null_check(x12, oopDesc::klass_offset_in_bytes());
3384   __ load_klass(x13, x12);
3385 
3386   Label no_such_method;
3387 
3388   // Preserve method for the throw_AbstractMethodErrorVerbose.
3389   __ mv(x28, xmethod);
3390   // Receiver subtype check against REFC.
3391   // Superklass in x10. Subklass in x13. Blows t1, x30
3392   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3393                              x13, x10, noreg,
3394                              // outputs: scan temp. reg, scan temp. reg
3395                              t1, x30,
3396                              no_such_interface,
3397                              /*return_method=*/false);
3398 
3399   // profile this call
3400   __ profile_virtual_call(x13, x30, x9);
3401 
3402   // Get declaring interface class from method, and itable index
3403   __ load_method_holder(x10, xmethod);
3404   __ lwu(xmethod, Address(xmethod, Method::itable_index_offset()));
3405   __ subw(xmethod, xmethod, Method::itable_index_max);
3406   __ negw(xmethod, xmethod);
3407 
3408   // Preserve recvKlass for throw_AbstractMethodErrorVerbose
3409   __ mv(xlocals, x13);
3410   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3411                              xlocals, x10, xmethod,
3412                              // outputs: method, scan temp. reg
3413                              xmethod, x30,
3414                              no_such_interface);
3415 
3416   // xmethod: Method to call
3417   // x12: receiver
3418   // Check for abstract method error
3419   // Note: This should be done more efficiently via a throw_abstract_method_error
3420   //       interpreter entry point and a conditional jump to it in case of a null
3421   //       method.
3422   __ beqz(xmethod, no_such_method);
3423 
3424   __ profile_arguments_type(x13, xmethod, x30, true);
3425 
3426   // do the call
3427   // x12: receiver
3428   // xmethod: Method
3429   __ jump_from_interpreted(xmethod);
3430   __ should_not_reach_here();
3431 
3432   // exception handling code follows ...
3433   // note: must restore interpreter registers to canonical
3434   //       state for exception handling to work correctly!
3435 
3436   __ bind(no_such_method);
3437   // throw exception
3438   __ restore_bcp();    // bcp must be correct for exception handler   (was destroyed)
3439   __ restore_locals(); // make sure locals pointer is correct as well (was destroyed)
3440   // Pass arguments for generating a verbose error message.
3441   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodErrorVerbose), x13, x28);
3442   // the call_VM checks for exception, so we should never return here.
3443   __ should_not_reach_here();
3444 
3445   __ bind(no_such_interface);
3446   // throw exceptiong
3447   __ restore_bcp();    // bcp must be correct for exception handler   (was destroyed)
3448   __ restore_locals(); // make sure locals pointer is correct as well (was destroyed)
3449   // Pass arguments for generating a verbose error message.
3450   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3451                                      InterpreterRuntime::throw_IncompatibleClassChangeErrorVerbose), x13, x10);
3452   // the call_VM checks for exception, so we should never return here.
3453   __ should_not_reach_here();
3454   return;
3455 }
3456 
3457 void TemplateTable::invokehandle(int byte_no) {
3458   transition(vtos, vtos);
3459   assert(byte_no == f1_byte, "use this argument");
3460 
3461   prepare_invoke(byte_no, xmethod, x10, x12);
3462   __ verify_method_ptr(x12);
3463   __ verify_oop(x12);
3464   __ null_check(x12);
3465 
3466   // FIXME: profile the LambdaForm also
3467 
3468   // x30 is safe to use here as a temp reg because it is about to
3469   // be clobbered by jump_from_interpreted().
3470   __ profile_final_call(x30);
3471   __ profile_arguments_type(x30, xmethod, x14, true);
3472 
3473   __ jump_from_interpreted(xmethod);
3474 }
3475 
3476 void TemplateTable::invokedynamic(int byte_no) {
3477   transition(vtos, vtos);
3478   assert(byte_no == f1_byte, "use this argument");
3479 
3480   prepare_invoke(byte_no, xmethod, x10);
3481 
3482   // x10: CallSite object (from cpool->resolved_references[])
3483   // xmethod: MH.linkToCallSite method (from f2)
3484 
3485   // Note: x10_callsite is already pushed by prepare_invoke
3486 
3487   // %%% should make a type profile for any invokedynamic that takes a ref argument
3488   // profile this call
3489   __ profile_call(xbcp);
3490   __ profile_arguments_type(x13, xmethod, x30, false);
3491 
3492   __ verify_oop(x10);
3493 
3494   __ jump_from_interpreted(xmethod);
3495 }
3496 
3497 //-----------------------------------------------------------------------------
3498 // Allocation
3499 
3500 void TemplateTable::_new() {
3501   transition(vtos, atos);
3502 
3503   __ get_unsigned_2_byte_index_at_bcp(x13, 1);
3504   Label slow_case;
3505   Label done;
3506   Label initialize_header;
3507   Label initialize_object; // including clearing the fields
3508 
3509   __ get_cpool_and_tags(x14, x10);
3510   // Make sure the class we're about to instantiate has been resolved.
3511   // This is done before loading InstanceKlass to be consistent with the order
3512   // how Constant Pool is update (see ConstantPool::klass_at_put)
3513   const int tags_offset = Array<u1>::base_offset_in_bytes();
3514   __ add(t0, x10, x13);
3515   __ la(t0, Address(t0, tags_offset));
3516   __ membar(MacroAssembler::AnyAny);
3517   __ lbu(t0, t0);
3518   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3519   __ sub(t1, t0, (u1)JVM_CONSTANT_Class);
3520   __ bnez(t1, slow_case);
3521 
3522   // get InstanceKlass
3523   __ load_resolved_klass_at_offset(x14, x13, x14, t0);
3524 
3525   // make sure klass is initialized & doesn't have finalizer
3526   // make sure klass is fully initialized
3527   __ lbu(t0, Address(x14, InstanceKlass::init_state_offset()));
3528   __ sub(t1, t0, (u1)InstanceKlass::fully_initialized);
3529   __ bnez(t1, slow_case);
3530 
3531   // get instance_size in InstanceKlass (scaled to a count of bytes)
3532   __ lwu(x13, Address(x14, Klass::layout_helper_offset()));
3533   // test to see if it has a finalizer or is malformed in some way
3534   __ andi(t0, x13, Klass::_lh_instance_slow_path_bit);
3535   __ bnez(t0, slow_case);
3536 
3537   // Allocate the instance:
3538   //  If TLAB is enabled:
3539   //    Try to allocate in the TLAB.
3540   //    If fails, go to the slow path.
3541   //  Else If inline contiguous allocations are enabled:
3542   //    Try to allocate in eden.
3543   //    If fails due to heap end, go to slow path
3544   //
3545   //  If TLAB is enabled OR inline contiguous is enabled:
3546   //    Initialize the allocation.
3547   //    Exit.
3548   //  Go to slow path.
3549   const bool allow_shared_alloc = Universe::heap()->supports_inline_contig_alloc();
3550 
3551   if (UseTLAB) {
3552     __ tlab_allocate(x10, x13, 0, noreg, x11, slow_case);
3553 
3554     if (ZeroTLAB) {
3555       // the fields have been already cleared
3556       __ j(initialize_header);
3557     } else {
3558       // initialize both the header and fields
3559       __ j(initialize_object);
3560     }
3561   } else {
3562     // Allocation in the shared Eden, if allowed.
3563     //
3564     // x13: instance size in bytes
3565     if (allow_shared_alloc) {
3566       __ eden_allocate(x10, x13, 0, x28, slow_case);
3567     }
3568   }
3569 
3570   // If USETLAB or allow_shared_alloc are true, the object is created above and
3571   // there is an initialized need. Otherwise, skip and go to the slow path.
3572   if (UseTLAB || allow_shared_alloc) {
3573     // The object is initialized before the header. If the object size is
3574     // zero, go directly to the header initialization.
3575     __ bind(initialize_object);
3576     __ sub(x13, x13, sizeof(oopDesc));
3577     __ beqz(x13, initialize_header);
3578 
3579     // Initialize obejct fields
3580     {
3581       __ add(x12, x10, sizeof(oopDesc));
3582       Label loop;
3583       __ bind(loop);
3584       __ sd(zr, Address(x12));
3585       __ add(x12, x12, BytesPerLong);
3586       __ sub(x13, x13, BytesPerLong);
3587       __ bnez(x13, loop);
3588     }
3589 
3590     // initialize object hader only.
3591     __ bind(initialize_header);
3592     __ mv(t0, (intptr_t)markWord::prototype().value());
3593     __ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes()));
3594     __ store_klass_gap(x10, zr);   // zero klass gap for compressed oops
3595     __ store_klass(x10, x14);      // store klass last
3596 
3597     {
3598       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3599       // Trigger dtrace event for fastpath
3600       __ push(atos); // save the return value
3601       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), x10);
3602       __ pop(atos); // restore the return value
3603     }
3604     __ j(done);
3605   }
3606 
3607   // slow case
3608   __ bind(slow_case);
3609   __ get_constant_pool(c_rarg1);
3610   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3611   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3612   __ verify_oop(x10);
3613 
3614   // continue
3615   __ bind(done);
3616   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3617   __ membar(MacroAssembler::StoreStore);
3618 }
3619 
3620 void TemplateTable::newarray() {
3621   transition(itos, atos);
3622   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3623   __ mv(c_rarg2, x10);
3624   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3625           c_rarg1, c_rarg2);
3626   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3627   __ membar(MacroAssembler::StoreStore);
3628 }
3629 
3630 void TemplateTable::anewarray() {
3631   transition(itos, atos);
3632   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3633   __ get_constant_pool(c_rarg1);
3634   __ mv(c_rarg3, x10);
3635   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3636           c_rarg1, c_rarg2, c_rarg3);
3637   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3638   __ membar(MacroAssembler::StoreStore);
3639 }
3640 
3641 void TemplateTable::arraylength() {
3642   transition(atos, itos);
3643   __ null_check(x10, arrayOopDesc::length_offset_in_bytes());
3644   __ lwu(x10, Address(x10, arrayOopDesc::length_offset_in_bytes()));
3645 }
3646 
3647 void TemplateTable::checkcast()
3648 {
3649   transition(atos, atos);
3650   Label done, is_null, ok_is_subtype, quicked, resolved;
3651   __ beqz(x10, is_null);
3652 
3653   // Get cpool & tags index
3654   __ get_cpool_and_tags(x12, x13); // x12=cpool, x13=tags array
3655   __ get_unsigned_2_byte_index_at_bcp(x9, 1); // x9=index
3656   // See if bytecode has already been quicked
3657   __ add(t0, x13, Array<u1>::base_offset_in_bytes());
3658   __ add(x11, t0, x9);
3659   __ membar(MacroAssembler::AnyAny);
3660   __ lbu(x11, x11);
3661   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3662   __ sub(t0, x11, (u1)JVM_CONSTANT_Class);
3663   __ beqz(t0, quicked);
3664 
3665   __ push(atos); // save receiver for result, and for GC
3666   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3667   // vm_result_2 has metadata result
3668   __ get_vm_result_2(x10, xthread);
3669   __ pop_reg(x13); // restore receiver
3670   __ j(resolved);
3671 
3672   // Get superklass in x10 and subklass in x13
3673   __ bind(quicked);
3674   __ mv(x13, x10); // Save object in x13; x10 needed for subtype check
3675   __ load_resolved_klass_at_offset(x12, x9, x10, t0); // x10 = klass
3676 
3677   __ bind(resolved);
3678   __ load_klass(x9, x13);
3679 
3680   // Generate subtype check.  Blows x12, x15.  Object in x13.
3681   // Superklass in x10.  Subklass in x9.
3682   __ gen_subtype_check(x9, ok_is_subtype);
3683 
3684   // Come here on failure
3685   __ push_reg(x13);
3686   // object is at TOS
3687   __ j(Interpreter::_throw_ClassCastException_entry);
3688 
3689   // Come here on success
3690   __ bind(ok_is_subtype);
3691   __ mv(x10, x13); // Restore object in x13
3692 
3693   // Collect counts on whether this test sees NULLs a lot or not.
3694   if (ProfileInterpreter) {
3695     __ j(done);
3696     __ bind(is_null);
3697     __ profile_null_seen(x12);
3698   } else {
3699     __ bind(is_null);   // same as 'done'
3700   }
3701   __ bind(done);
3702 }
3703 
3704 void TemplateTable::instanceof() {
3705   transition(atos, itos);
3706   Label done, is_null, ok_is_subtype, quicked, resolved;
3707   __ beqz(x10, is_null);
3708 
3709   // Get cpool & tags index
3710   __ get_cpool_and_tags(x12, x13); // x12=cpool, x13=tags array
3711   __ get_unsigned_2_byte_index_at_bcp(x9, 1); // x9=index
3712   // See if bytecode has already been quicked
3713   __ add(t0, x13, Array<u1>::base_offset_in_bytes());
3714   __ add(x11, t0, x9);
3715   __ membar(MacroAssembler::AnyAny);
3716   __ lbu(x11, x11);
3717   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3718   __ sub(t0, x11, (u1)JVM_CONSTANT_Class);
3719   __ beqz(t0, quicked);
3720 
3721   __ push(atos); // save receiver for result, and for GC
3722   call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3723   // vm_result_2 has metadata result
3724   __ get_vm_result_2(x10, xthread);
3725   __ pop_reg(x13); // restore receiver
3726   __ verify_oop(x13);
3727   __ load_klass(x13, x13);
3728   __ j(resolved);
3729 
3730   // Get superklass in x10 and subklass in x13
3731   __ bind(quicked);
3732   __ load_klass(x13, x10);
3733   __ load_resolved_klass_at_offset(x12, x9, x10, t0);
3734 
3735   __ bind(resolved);
3736 
3737   // Generate subtype check.  Blows x12, x15
3738   // Superklass in x10.  Subklass in x13.
3739   __ gen_subtype_check(x13, ok_is_subtype);
3740 
3741   // Come here on failure
3742   __ mv(x10, zr);
3743   __ j(done);
3744   // Come here on success
3745   __ bind(ok_is_subtype);
3746   __ li(x10, 1);
3747 
3748   // Collect counts on whether this test sees NULLs a lot or not.
3749   if (ProfileInterpreter) {
3750     __ j(done);
3751     __ bind(is_null);
3752     __ profile_null_seen(x12);
3753   } else {
3754     __ bind(is_null);   // same as 'done'
3755   }
3756   __ bind(done);
3757   // x10 = 0: obj == NULL or  obj is not an instanceof the specified klass
3758   // x10 = 1: obj != NULL and obj is     an instanceof the specified klass
3759 }
3760 
3761 //-----------------------------------------------------------------------------
3762 // Breakpoints
3763 void TemplateTable::_breakpoint() {
3764   // Note: We get here even if we are single stepping..
3765   // jbug inists on setting breakpoints at every bytecode
3766   // even if we are in single step mode.
3767 
3768   transition(vtos, vtos);
3769 
3770   // get the unpatched byte code
3771   __ get_method(c_rarg1);
3772   __ call_VM(noreg,
3773              CAST_FROM_FN_PTR(address,
3774                               InterpreterRuntime::get_original_bytecode_at),
3775              c_rarg1, xbcp);
3776   __ mv(x9, x10);
3777 
3778   // post the breakpoint event
3779   __ call_VM(noreg,
3780              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
3781              xmethod, xbcp);
3782 
3783   // complete the execution of original bytecode
3784   __ mv(t0, x9);
3785   __ dispatch_only_normal(vtos);
3786 }
3787 
3788 //-----------------------------------------------------------------------------
3789 // Exceptions
3790 
3791 void TemplateTable::athrow() {
3792   transition(atos, vtos);
3793   __ null_check(x10);
3794   __ j(Interpreter::throw_exception_entry());
3795 }
3796 
3797 //-----------------------------------------------------------------------------
3798 // Synchronization
3799 //
3800 // Note: monitorenter & exit are symmetric routines; which is reflected
3801 //       in the assembly code structure as well
3802 //
3803 // Stack layout:
3804 //
3805 // [expressions  ] <--- esp               = expression stack top
3806 // ..
3807 // [expressions  ]
3808 // [monitor entry] <--- monitor block top = expression stack bot
3809 // ..
3810 // [monitor entry]
3811 // [frame data   ] <--- monitor block bot
3812 // ...
3813 // [saved fp     ] <--- fp
3814 void TemplateTable::monitorenter()
3815 {
3816   transition(atos, vtos);
3817 
3818    // check for NULL object
3819    __ null_check(x10);
3820 
3821    const Address monitor_block_top(
3822          fp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3823    const Address monitor_block_bot(
3824          fp, frame::interpreter_frame_initial_sp_offset * wordSize);
3825    const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3826 
3827    Label allocated;
3828 
3829    // initialize entry pointer
3830    __ mv(c_rarg1, zr); // points to free slot or NULL
3831 
3832    // find a free slot in the monitor block (result in c_rarg1)
3833    {
3834      Label entry, loop, exit, notUsed;
3835      __ ld(c_rarg3, monitor_block_top); // points to current entry,
3836                                         // starting with top-most entry
3837      __ la(c_rarg2, monitor_block_bot); // points to word before bottom
3838 
3839      __ j(entry);
3840 
3841      __ bind(loop);
3842      // check if current entry is used
3843      // if not used then remember entry in c_rarg1
3844      __ ld(t0, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
3845      __ bnez(t0, notUsed);
3846      __ mv(c_rarg1, c_rarg3);
3847      __ bind(notUsed);
3848      // check if current entry is for same object
3849      // if same object then stop searching
3850      __ beq(x10, t0, exit);
3851      // otherwise advance to next entry
3852      __ add(c_rarg3, c_rarg3, entry_size);
3853      __ bind(entry);
3854      // check if bottom reached
3855      // if not at bottom then check this entry
3856      __ bne(c_rarg3, c_rarg2, loop);
3857      __ bind(exit);
3858    }
3859 
3860    __ bnez(c_rarg1, allocated); // check if a slot has been found and
3861                              // if found, continue with that on
3862 
3863    // allocate one if there's no free slot
3864    {
3865      Label entry, loop;
3866      // 1. compute new pointers            // esp: old expression stack top
3867      __ ld(c_rarg1, monitor_block_bot);    // c_rarg1: old expression stack bottom
3868      __ sub(esp, esp, entry_size);         // move expression stack top
3869      __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
3870      __ mv(c_rarg3, esp);                  // set start value for copy loop
3871      __ sd(c_rarg1, monitor_block_bot);    // set new monitor block bottom
3872      __ sub(sp, sp, entry_size);           // make room for the monitor
3873 
3874      __ j(entry);
3875      // 2. move expression stack contents
3876      __ bind(loop);
3877      __ ld(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
3878                                                    // word from old location
3879      __ sd(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
3880      __ add(c_rarg3, c_rarg3, wordSize);           // advance to next word
3881      __ bind(entry);
3882      __ bne(c_rarg3, c_rarg1, loop);    // check if bottom reached.if not at bottom
3883                                         // then copy next word
3884    }
3885 
3886    // call run-time routine
3887    // c_rarg1: points to monitor entry
3888    __ bind(allocated);
3889 
3890    // Increment bcp to point to the next bytecode, so exception
3891    // handling for async. exceptions work correctly.
3892    // The object has already been poped from the stack, so the
3893    // expression stack looks correct.
3894    __ addi(xbcp, xbcp, 1);
3895 
3896    // store object
3897    __ sd(x10, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3898    __ lock_object(c_rarg1);
3899 
3900    // check to make sure this monitor doesn't cause stack overflow after locking
3901    __ save_bcp();  // in case of exception
3902    __ generate_stack_overflow_check(0);
3903 
3904    // The bcp has already been incremented. Just need to dispatch to
3905    // next instruction.
3906    __ dispatch_next(vtos);
3907 }
3908 
3909 void TemplateTable::monitorexit()
3910 {
3911   transition(atos, vtos);
3912 
3913   // check for NULL object
3914   __ null_check(x10);
3915 
3916   const Address monitor_block_top(
3917         fp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3918   const Address monitor_block_bot(
3919         fp, frame::interpreter_frame_initial_sp_offset * wordSize);
3920   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3921 
3922   Label found;
3923 
3924   // find matching slot
3925   {
3926     Label entry, loop;
3927     __ ld(c_rarg1, monitor_block_top); // points to current entry,
3928                                         // starting with top-most entry
3929     __ la(c_rarg2, monitor_block_bot); // points to word before bottom
3930                                         // of monitor block
3931     __ j(entry);
3932 
3933     __ bind(loop);
3934     // check if current entry is for same object
3935     __ ld(t0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3936     // if same object then stop searching
3937     __ beq(x10, t0, found);
3938     // otherwise advance to next entry
3939     __ add(c_rarg1, c_rarg1, entry_size);
3940     __ bind(entry);
3941     // check if bottom reached
3942     // if not at bottom then check this entry
3943     __ bne(c_rarg1, c_rarg2, loop);
3944   }
3945 
3946   // error handling. Unlocking was not block-structured
3947   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3948                    InterpreterRuntime::throw_illegal_monitor_state_exception));
3949   __ should_not_reach_here();
3950 
3951   // call run-time routine
3952   __ bind(found);
3953   __ push_ptr(x10); // make sure object is on stack (contract with oopMaps)
3954   __ unlock_object(c_rarg1);
3955   __ pop_ptr(x10); // discard object
3956 }
3957 
3958 // Wide instructions
3959 void TemplateTable::wide()
3960 {
3961   __ load_unsigned_byte(x9, at_bcp(1));
3962   __ mv(t0, (address)Interpreter::_wentry_point);
3963   __ slli(t1, x9, 3);
3964   __ add(t0, t1, t0);
3965   __ ld(t0, Address(t0));
3966   __ jr(t0);
3967 }
3968 
3969 // Multi arrays
3970 void TemplateTable::multianewarray() {
3971   transition(vtos, atos);
3972   __ load_unsigned_byte(x10, at_bcp(3)); // get number of dimensions
3973   // last dim is on top of stack; we want address of first one:
3974   // first_addr = last_addr + (ndims - 1) * wordSize
3975   __ slli(c_rarg1, x10, 3);
3976   __ add(c_rarg1, c_rarg1, esp);
3977   __ sub(c_rarg1, c_rarg1, wordSize);
3978   call_VM(x10,
3979           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
3980           c_rarg1);
3981   __ load_unsigned_byte(x11, at_bcp(3));
3982   __ slli(t0, x11, 3);
3983   __ add(esp, esp, t0);
3984 }