1 /*
   2  * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "compiler/disassembler.hpp"
  28 #include "gc/shared/collectedHeap.hpp"
  29 #include "gc/shared/gc_globals.hpp"
  30 #include "gc/shared/tlab_globals.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "interpreter/interpreterRuntime.hpp"
  33 #include "interpreter/interp_masm.hpp"
  34 #include "interpreter/templateTable.hpp"
  35 #include "memory/universe.hpp"
  36 #include "oops/methodCounters.hpp"
  37 #include "oops/methodData.hpp"
  38 #include "oops/objArrayKlass.hpp"
  39 #include "oops/oop.inline.hpp"
  40 #include "oops/inlineKlass.hpp"
  41 #include "oops/resolvedFieldEntry.hpp"
  42 #include "oops/resolvedIndyEntry.hpp"
  43 #include "oops/resolvedMethodEntry.hpp"
  44 #include "prims/jvmtiExport.hpp"
  45 #include "prims/methodHandles.hpp"
  46 #include "runtime/frame.inline.hpp"
  47 #include "runtime/safepointMechanism.hpp"
  48 #include "runtime/sharedRuntime.hpp"
  49 #include "runtime/stubRoutines.hpp"
  50 #include "runtime/synchronizer.hpp"
  51 #include "utilities/macros.hpp"
  52 
  53 #define __ Disassembler::hook<InterpreterMacroAssembler>(__FILE__, __LINE__, _masm)->
  54 
  55 // Global Register Names
  56 static const Register rbcp     = LP64_ONLY(r13) NOT_LP64(rsi);
  57 static const Register rlocals  = LP64_ONLY(r14) NOT_LP64(rdi);
  58 
  59 // Address Computation: local variables
  60 static inline Address iaddress(int n) {
  61   return Address(rlocals, Interpreter::local_offset_in_bytes(n));
  62 }
  63 
  64 static inline Address laddress(int n) {
  65   return iaddress(n + 1);
  66 }
  67 
  68 #ifndef _LP64
  69 static inline Address haddress(int n) {
  70   return iaddress(n + 0);
  71 }
  72 #endif
  73 
  74 static inline Address faddress(int n) {
  75   return iaddress(n);
  76 }
  77 
  78 static inline Address daddress(int n) {
  79   return laddress(n);
  80 }
  81 
  82 static inline Address aaddress(int n) {
  83   return iaddress(n);
  84 }
  85 
  86 static inline Address iaddress(Register r) {
  87   return Address(rlocals, r, Address::times_ptr);
  88 }
  89 
  90 static inline Address laddress(Register r) {
  91   return Address(rlocals, r, Address::times_ptr, Interpreter::local_offset_in_bytes(1));
  92 }
  93 
  94 #ifndef _LP64
  95 static inline Address haddress(Register r)       {
  96   return Address(rlocals, r, Interpreter::stackElementScale(), Interpreter::local_offset_in_bytes(0));
  97 }
  98 #endif
  99 
 100 static inline Address faddress(Register r) {
 101   return iaddress(r);
 102 }
 103 
 104 static inline Address daddress(Register r) {
 105   return laddress(r);
 106 }
 107 
 108 static inline Address aaddress(Register r) {
 109   return iaddress(r);
 110 }
 111 
 112 
 113 // expression stack
 114 // (Note: Must not use symmetric equivalents at_rsp_m1/2 since they store
 115 // data beyond the rsp which is potentially unsafe in an MT environment;
 116 // an interrupt may overwrite that data.)
 117 static inline Address at_rsp   () {
 118   return Address(rsp, 0);
 119 }
 120 
 121 // At top of Java expression stack which may be different than esp().  It
 122 // isn't for category 1 objects.
 123 static inline Address at_tos   () {
 124   return Address(rsp,  Interpreter::expr_offset_in_bytes(0));
 125 }
 126 
 127 static inline Address at_tos_p1() {
 128   return Address(rsp,  Interpreter::expr_offset_in_bytes(1));
 129 }
 130 
 131 static inline Address at_tos_p2() {
 132   return Address(rsp,  Interpreter::expr_offset_in_bytes(2));
 133 }
 134 
 135 // Condition conversion
 136 static Assembler::Condition j_not(TemplateTable::Condition cc) {
 137   switch (cc) {
 138   case TemplateTable::equal        : return Assembler::notEqual;
 139   case TemplateTable::not_equal    : return Assembler::equal;
 140   case TemplateTable::less         : return Assembler::greaterEqual;
 141   case TemplateTable::less_equal   : return Assembler::greater;
 142   case TemplateTable::greater      : return Assembler::lessEqual;
 143   case TemplateTable::greater_equal: return Assembler::less;
 144   }
 145   ShouldNotReachHere();
 146   return Assembler::zero;
 147 }
 148 
 149 
 150 
 151 // Miscellaneous helper routines
 152 // Store an oop (or null) at the address described by obj.
 153 // If val == noreg this means store a null
 154 
 155 
 156 static void do_oop_store(InterpreterMacroAssembler* _masm,
 157                          Address dst,
 158                          Register val,
 159                          DecoratorSet decorators = 0) {
 160   assert(val == noreg || val == rax, "parameter is just for looks");
 161   __ store_heap_oop(dst, val,
 162                     NOT_LP64(rdx) LP64_ONLY(rscratch2),
 163                     NOT_LP64(rbx) LP64_ONLY(r9),
 164                     NOT_LP64(rsi) LP64_ONLY(r8), decorators);
 165 }
 166 
 167 static void do_oop_load(InterpreterMacroAssembler* _masm,
 168                         Address src,
 169                         Register dst,
 170                         DecoratorSet decorators = 0) {
 171   __ load_heap_oop(dst, src, rdx, rbx, decorators);
 172 }
 173 
 174 Address TemplateTable::at_bcp(int offset) {
 175   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 176   return Address(rbcp, offset);
 177 }
 178 
 179 
 180 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 181                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 182                                    int byte_no) {
 183   if (!RewriteBytecodes)  return;
 184   Label L_patch_done;
 185 
 186   switch (bc) {
 187   case Bytecodes::_fast_vputfield:
 188   case Bytecodes::_fast_aputfield:
 189   case Bytecodes::_fast_bputfield:
 190   case Bytecodes::_fast_zputfield:
 191   case Bytecodes::_fast_cputfield:
 192   case Bytecodes::_fast_dputfield:
 193   case Bytecodes::_fast_fputfield:
 194   case Bytecodes::_fast_iputfield:
 195   case Bytecodes::_fast_lputfield:
 196   case Bytecodes::_fast_sputfield:
 197     {
 198       // We skip bytecode quickening for putfield instructions when
 199       // the put_code written to the constant pool cache is zero.
 200       // This is required so that every execution of this instruction
 201       // calls out to InterpreterRuntime::resolve_get_put to do
 202       // additional, required work.
 203       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 204       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 205       __ load_field_entry(temp_reg, bc_reg);
 206       if (byte_no == f1_byte) {
 207         __ load_unsigned_byte(temp_reg, Address(temp_reg, in_bytes(ResolvedFieldEntry::get_code_offset())));
 208       } else {
 209         __ load_unsigned_byte(temp_reg, Address(temp_reg, in_bytes(ResolvedFieldEntry::put_code_offset())));
 210       }
 211 
 212       __ movl(bc_reg, bc);
 213       __ cmpl(temp_reg, (int) 0);
 214       __ jcc(Assembler::zero, L_patch_done);  // don't patch
 215     }
 216     break;
 217   default:
 218     assert(byte_no == -1, "sanity");
 219     // the pair bytecodes have already done the load.
 220     if (load_bc_into_bc_reg) {
 221       __ movl(bc_reg, bc);
 222     }
 223   }
 224 
 225   if (JvmtiExport::can_post_breakpoint()) {
 226     Label L_fast_patch;
 227     // if a breakpoint is present we can't rewrite the stream directly
 228     __ movzbl(temp_reg, at_bcp(0));
 229     __ cmpl(temp_reg, Bytecodes::_breakpoint);
 230     __ jcc(Assembler::notEqual, L_fast_patch);
 231     __ get_method(temp_reg);
 232     // Let breakpoint table handling rewrite to quicker bytecode
 233     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), temp_reg, rbcp, bc_reg);
 234 #ifndef ASSERT
 235     __ jmpb(L_patch_done);
 236 #else
 237     __ jmp(L_patch_done);
 238 #endif
 239     __ bind(L_fast_patch);
 240   }
 241 
 242 #ifdef ASSERT
 243   Label L_okay;
 244   __ load_unsigned_byte(temp_reg, at_bcp(0));
 245   __ cmpl(temp_reg, (int) Bytecodes::java_code(bc));
 246   __ jcc(Assembler::equal, L_okay);
 247   __ cmpl(temp_reg, bc_reg);
 248   __ jcc(Assembler::equal, L_okay);
 249   __ stop("patching the wrong bytecode");
 250   __ bind(L_okay);
 251 #endif
 252 
 253   // patch bytecode
 254   __ movb(at_bcp(0), bc_reg);
 255   __ bind(L_patch_done);
 256 }
 257 // Individual instructions
 258 
 259 
 260 void TemplateTable::nop() {
 261   transition(vtos, vtos);
 262   // nothing to do
 263 }
 264 
 265 void TemplateTable::shouldnotreachhere() {
 266   transition(vtos, vtos);
 267   __ stop("shouldnotreachhere bytecode");
 268 }
 269 
 270 void TemplateTable::aconst_null() {
 271   transition(vtos, atos);
 272   __ xorl(rax, rax);
 273 }
 274 
 275 void TemplateTable::iconst(int value) {
 276   transition(vtos, itos);
 277   if (value == 0) {
 278     __ xorl(rax, rax);
 279   } else {
 280     __ movl(rax, value);
 281   }
 282 }
 283 
 284 void TemplateTable::lconst(int value) {
 285   transition(vtos, ltos);
 286   if (value == 0) {
 287     __ xorl(rax, rax);
 288   } else {
 289     __ movl(rax, value);
 290   }
 291 #ifndef _LP64
 292   assert(value >= 0, "check this code");
 293   __ xorptr(rdx, rdx);
 294 #endif
 295 }
 296 
 297 
 298 
 299 void TemplateTable::fconst(int value) {
 300   transition(vtos, ftos);
 301   if (UseSSE >= 1) {
 302     static float one = 1.0f, two = 2.0f;
 303     switch (value) {
 304     case 0:
 305       __ xorps(xmm0, xmm0);
 306       break;
 307     case 1:
 308       __ movflt(xmm0, ExternalAddress((address) &one), rscratch1);
 309       break;
 310     case 2:
 311       __ movflt(xmm0, ExternalAddress((address) &two), rscratch1);
 312       break;
 313     default:
 314       ShouldNotReachHere();
 315       break;
 316     }
 317   } else {
 318 #ifdef _LP64
 319     ShouldNotReachHere();
 320 #else
 321            if (value == 0) { __ fldz();
 322     } else if (value == 1) { __ fld1();
 323     } else if (value == 2) { __ fld1(); __ fld1(); __ faddp(); // should do a better solution here
 324     } else                 { ShouldNotReachHere();
 325     }
 326 #endif // _LP64
 327   }
 328 }
 329 
 330 void TemplateTable::dconst(int value) {
 331   transition(vtos, dtos);
 332   if (UseSSE >= 2) {
 333     static double one = 1.0;
 334     switch (value) {
 335     case 0:
 336       __ xorpd(xmm0, xmm0);
 337       break;
 338     case 1:
 339       __ movdbl(xmm0, ExternalAddress((address) &one), rscratch1);
 340       break;
 341     default:
 342       ShouldNotReachHere();
 343       break;
 344     }
 345   } else {
 346 #ifdef _LP64
 347     ShouldNotReachHere();
 348 #else
 349            if (value == 0) { __ fldz();
 350     } else if (value == 1) { __ fld1();
 351     } else                 { ShouldNotReachHere();
 352     }
 353 #endif
 354   }
 355 }
 356 
 357 void TemplateTable::bipush() {
 358   transition(vtos, itos);
 359   __ load_signed_byte(rax, at_bcp(1));
 360 }
 361 
 362 void TemplateTable::sipush() {
 363   transition(vtos, itos);
 364   __ load_unsigned_short(rax, at_bcp(1));
 365   __ bswapl(rax);
 366   __ sarl(rax, 16);
 367 }
 368 
 369 void TemplateTable::ldc(LdcType type) {
 370   transition(vtos, vtos);
 371   Register rarg = NOT_LP64(rcx) LP64_ONLY(c_rarg1);
 372   Label call_ldc, notFloat, notClass, notInt, Done;
 373 
 374   if (is_ldc_wide(type)) {
 375     __ get_unsigned_2_byte_index_at_bcp(rbx, 1);
 376   } else {
 377     __ load_unsigned_byte(rbx, at_bcp(1));
 378   }
 379 
 380   __ get_cpool_and_tags(rcx, rax);
 381   const int base_offset = ConstantPool::header_size() * wordSize;
 382   const int tags_offset = Array<u1>::base_offset_in_bytes();
 383 
 384   // get type
 385   __ movzbl(rdx, Address(rax, rbx, Address::times_1, tags_offset));
 386 
 387   // unresolved class - get the resolved class
 388   __ cmpl(rdx, JVM_CONSTANT_UnresolvedClass);
 389   __ jccb(Assembler::equal, call_ldc);
 390 
 391   // unresolved class in error state - call into runtime to throw the error
 392   // from the first resolution attempt
 393   __ cmpl(rdx, JVM_CONSTANT_UnresolvedClassInError);
 394   __ jccb(Assembler::equal, call_ldc);
 395 
 396   // resolved class - need to call vm to get java mirror of the class
 397   __ cmpl(rdx, JVM_CONSTANT_Class);
 398   __ jcc(Assembler::notEqual, notClass);
 399 
 400   __ bind(call_ldc);
 401 
 402   __ movl(rarg, is_ldc_wide(type) ? 1 : 0);
 403   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), rarg);
 404 
 405   __ push(atos);
 406   __ jmp(Done);
 407 
 408   __ bind(notClass);
 409   __ cmpl(rdx, JVM_CONSTANT_Float);
 410   __ jccb(Assembler::notEqual, notFloat);
 411 
 412   // ftos
 413   __ load_float(Address(rcx, rbx, Address::times_ptr, base_offset));
 414   __ push(ftos);
 415   __ jmp(Done);
 416 
 417   __ bind(notFloat);
 418   __ cmpl(rdx, JVM_CONSTANT_Integer);
 419   __ jccb(Assembler::notEqual, notInt);
 420 
 421   // itos
 422   __ movl(rax, Address(rcx, rbx, Address::times_ptr, base_offset));
 423   __ push(itos);
 424   __ jmp(Done);
 425 
 426   // assume the tag is for condy; if not, the VM runtime will tell us
 427   __ bind(notInt);
 428   condy_helper(Done);
 429 
 430   __ bind(Done);
 431 }
 432 
 433 // Fast path for caching oop constants.
 434 void TemplateTable::fast_aldc(LdcType type) {
 435   transition(vtos, atos);
 436 
 437   Register result = rax;
 438   Register tmp = rdx;
 439   Register rarg = NOT_LP64(rcx) LP64_ONLY(c_rarg1);
 440   int index_size = is_ldc_wide(type) ? sizeof(u2) : sizeof(u1);
 441 
 442   Label resolved;
 443 
 444   // We are resolved if the resolved reference cache entry contains a
 445   // non-null object (String, MethodType, etc.)
 446   assert_different_registers(result, tmp);
 447   __ get_cache_index_at_bcp(tmp, 1, index_size);
 448   __ load_resolved_reference_at_index(result, tmp);
 449   __ testptr(result, result);
 450   __ jcc(Assembler::notZero, resolved);
 451 
 452   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 453 
 454   // first time invocation - must resolve first
 455   __ movl(rarg, (int)bytecode());
 456   __ call_VM(result, entry, rarg);
 457   __ bind(resolved);
 458 
 459   { // Check for the null sentinel.
 460     // If we just called the VM, it already did the mapping for us,
 461     // but it's harmless to retry.
 462     Label notNull;
 463     ExternalAddress null_sentinel((address)Universe::the_null_sentinel_addr());
 464     __ movptr(tmp, null_sentinel);
 465     __ resolve_oop_handle(tmp, rscratch2);
 466     __ cmpoop(tmp, result);
 467     __ jccb(Assembler::notEqual, notNull);
 468     __ xorptr(result, result);  // null object reference
 469     __ bind(notNull);
 470   }
 471 
 472   if (VerifyOops) {
 473     __ verify_oop(result);
 474   }
 475 }
 476 
 477 void TemplateTable::ldc2_w() {
 478   transition(vtos, vtos);
 479   Label notDouble, notLong, Done;
 480   __ get_unsigned_2_byte_index_at_bcp(rbx, 1);
 481 
 482   __ get_cpool_and_tags(rcx, rax);
 483   const int base_offset = ConstantPool::header_size() * wordSize;
 484   const int tags_offset = Array<u1>::base_offset_in_bytes();
 485 
 486   // get type
 487   __ movzbl(rdx, Address(rax, rbx, Address::times_1, tags_offset));
 488   __ cmpl(rdx, JVM_CONSTANT_Double);
 489   __ jccb(Assembler::notEqual, notDouble);
 490 
 491   // dtos
 492   __ load_double(Address(rcx, rbx, Address::times_ptr, base_offset));
 493   __ push(dtos);
 494 
 495   __ jmp(Done);
 496   __ bind(notDouble);
 497   __ cmpl(rdx, JVM_CONSTANT_Long);
 498   __ jccb(Assembler::notEqual, notLong);
 499 
 500   // ltos
 501   __ movptr(rax, Address(rcx, rbx, Address::times_ptr, base_offset + 0 * wordSize));
 502   NOT_LP64(__ movptr(rdx, Address(rcx, rbx, Address::times_ptr, base_offset + 1 * wordSize)));
 503   __ push(ltos);
 504   __ jmp(Done);
 505 
 506   __ bind(notLong);
 507   condy_helper(Done);
 508 
 509   __ bind(Done);
 510 }
 511 
 512 void TemplateTable::condy_helper(Label& Done) {
 513   const Register obj = rax;
 514   const Register off = rbx;
 515   const Register flags = rcx;
 516   const Register rarg = NOT_LP64(rcx) LP64_ONLY(c_rarg1);
 517   __ movl(rarg, (int)bytecode());
 518   call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc), rarg);
 519 #ifndef _LP64
 520   // borrow rdi from locals
 521   __ get_thread(rdi);
 522   __ get_vm_result_2(flags, rdi);
 523   __ restore_locals();
 524 #else
 525   __ get_vm_result_2(flags, r15_thread);
 526 #endif
 527   // VMr = obj = base address to find primitive value to push
 528   // VMr2 = flags = (tos, off) using format of CPCE::_flags
 529   __ movl(off, flags);
 530   __ andl(off, ConstantPoolCache::field_index_mask);
 531   const Address field(obj, off, Address::times_1, 0*wordSize);
 532 
 533   // What sort of thing are we loading?
 534   __ shrl(flags, ConstantPoolCache::tos_state_shift);
 535   __ andl(flags, ConstantPoolCache::tos_state_mask);
 536 
 537   switch (bytecode()) {
 538   case Bytecodes::_ldc:
 539   case Bytecodes::_ldc_w:
 540     {
 541       // tos in (itos, ftos, stos, btos, ctos, ztos)
 542       Label notInt, notFloat, notShort, notByte, notChar, notBool;
 543       __ cmpl(flags, itos);
 544       __ jccb(Assembler::notEqual, notInt);
 545       // itos
 546       __ movl(rax, field);
 547       __ push(itos);
 548       __ jmp(Done);
 549 
 550       __ bind(notInt);
 551       __ cmpl(flags, ftos);
 552       __ jccb(Assembler::notEqual, notFloat);
 553       // ftos
 554       __ load_float(field);
 555       __ push(ftos);
 556       __ jmp(Done);
 557 
 558       __ bind(notFloat);
 559       __ cmpl(flags, stos);
 560       __ jccb(Assembler::notEqual, notShort);
 561       // stos
 562       __ load_signed_short(rax, field);
 563       __ push(stos);
 564       __ jmp(Done);
 565 
 566       __ bind(notShort);
 567       __ cmpl(flags, btos);
 568       __ jccb(Assembler::notEqual, notByte);
 569       // btos
 570       __ load_signed_byte(rax, field);
 571       __ push(btos);
 572       __ jmp(Done);
 573 
 574       __ bind(notByte);
 575       __ cmpl(flags, ctos);
 576       __ jccb(Assembler::notEqual, notChar);
 577       // ctos
 578       __ load_unsigned_short(rax, field);
 579       __ push(ctos);
 580       __ jmp(Done);
 581 
 582       __ bind(notChar);
 583       __ cmpl(flags, ztos);
 584       __ jccb(Assembler::notEqual, notBool);
 585       // ztos
 586       __ load_signed_byte(rax, field);
 587       __ push(ztos);
 588       __ jmp(Done);
 589 
 590       __ bind(notBool);
 591       break;
 592     }
 593 
 594   case Bytecodes::_ldc2_w:
 595     {
 596       Label notLong, notDouble;
 597       __ cmpl(flags, ltos);
 598       __ jccb(Assembler::notEqual, notLong);
 599       // ltos
 600       // Loading high word first because movptr clobbers rax
 601       NOT_LP64(__ movptr(rdx, field.plus_disp(4)));
 602       __ movptr(rax, field);
 603       __ push(ltos);
 604       __ jmp(Done);
 605 
 606       __ bind(notLong);
 607       __ cmpl(flags, dtos);
 608       __ jccb(Assembler::notEqual, notDouble);
 609       // dtos
 610       __ load_double(field);
 611       __ push(dtos);
 612       __ jmp(Done);
 613 
 614       __ bind(notDouble);
 615       break;
 616     }
 617 
 618   default:
 619     ShouldNotReachHere();
 620   }
 621 
 622   __ stop("bad ldc/condy");
 623 }
 624 
 625 void TemplateTable::locals_index(Register reg, int offset) {
 626   __ load_unsigned_byte(reg, at_bcp(offset));
 627   __ negptr(reg);
 628 }
 629 
 630 void TemplateTable::iload() {
 631   iload_internal();
 632 }
 633 
 634 void TemplateTable::nofast_iload() {
 635   iload_internal(may_not_rewrite);
 636 }
 637 
 638 void TemplateTable::iload_internal(RewriteControl rc) {
 639   transition(vtos, itos);
 640   if (RewriteFrequentPairs && rc == may_rewrite) {
 641     Label rewrite, done;
 642     const Register bc = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
 643     LP64_ONLY(assert(rbx != bc, "register damaged"));
 644 
 645     // get next byte
 646     __ load_unsigned_byte(rbx,
 647                           at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 648     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 649     // last two iloads in a pair.  Comparing against fast_iload means that
 650     // the next bytecode is neither an iload or a caload, and therefore
 651     // an iload pair.
 652     __ cmpl(rbx, Bytecodes::_iload);
 653     __ jcc(Assembler::equal, done);
 654 
 655     __ cmpl(rbx, Bytecodes::_fast_iload);
 656     __ movl(bc, Bytecodes::_fast_iload2);
 657 
 658     __ jccb(Assembler::equal, rewrite);
 659 
 660     // if _caload, rewrite to fast_icaload
 661     __ cmpl(rbx, Bytecodes::_caload);
 662     __ movl(bc, Bytecodes::_fast_icaload);
 663     __ jccb(Assembler::equal, rewrite);
 664 
 665     // rewrite so iload doesn't check again.
 666     __ movl(bc, Bytecodes::_fast_iload);
 667 
 668     // rewrite
 669     // bc: fast bytecode
 670     __ bind(rewrite);
 671     patch_bytecode(Bytecodes::_iload, bc, rbx, false);
 672     __ bind(done);
 673   }
 674 
 675   // Get the local value into tos
 676   locals_index(rbx);
 677   __ movl(rax, iaddress(rbx));
 678 }
 679 
 680 void TemplateTable::fast_iload2() {
 681   transition(vtos, itos);
 682   locals_index(rbx);
 683   __ movl(rax, iaddress(rbx));
 684   __ push(itos);
 685   locals_index(rbx, 3);
 686   __ movl(rax, iaddress(rbx));
 687 }
 688 
 689 void TemplateTable::fast_iload() {
 690   transition(vtos, itos);
 691   locals_index(rbx);
 692   __ movl(rax, iaddress(rbx));
 693 }
 694 
 695 void TemplateTable::lload() {
 696   transition(vtos, ltos);
 697   locals_index(rbx);
 698   __ movptr(rax, laddress(rbx));
 699   NOT_LP64(__ movl(rdx, haddress(rbx)));
 700 }
 701 
 702 void TemplateTable::fload() {
 703   transition(vtos, ftos);
 704   locals_index(rbx);
 705   __ load_float(faddress(rbx));
 706 }
 707 
 708 void TemplateTable::dload() {
 709   transition(vtos, dtos);
 710   locals_index(rbx);
 711   __ load_double(daddress(rbx));
 712 }
 713 
 714 void TemplateTable::aload() {
 715   transition(vtos, atos);
 716   locals_index(rbx);
 717   __ movptr(rax, aaddress(rbx));
 718 }
 719 
 720 void TemplateTable::locals_index_wide(Register reg) {
 721   __ load_unsigned_short(reg, at_bcp(2));
 722   __ bswapl(reg);
 723   __ shrl(reg, 16);
 724   __ negptr(reg);
 725 }
 726 
 727 void TemplateTable::wide_iload() {
 728   transition(vtos, itos);
 729   locals_index_wide(rbx);
 730   __ movl(rax, iaddress(rbx));
 731 }
 732 
 733 void TemplateTable::wide_lload() {
 734   transition(vtos, ltos);
 735   locals_index_wide(rbx);
 736   __ movptr(rax, laddress(rbx));
 737   NOT_LP64(__ movl(rdx, haddress(rbx)));
 738 }
 739 
 740 void TemplateTable::wide_fload() {
 741   transition(vtos, ftos);
 742   locals_index_wide(rbx);
 743   __ load_float(faddress(rbx));
 744 }
 745 
 746 void TemplateTable::wide_dload() {
 747   transition(vtos, dtos);
 748   locals_index_wide(rbx);
 749   __ load_double(daddress(rbx));
 750 }
 751 
 752 void TemplateTable::wide_aload() {
 753   transition(vtos, atos);
 754   locals_index_wide(rbx);
 755   __ movptr(rax, aaddress(rbx));
 756 }
 757 
 758 void TemplateTable::index_check(Register array, Register index) {
 759   // Pop ptr into array
 760   __ pop_ptr(array);
 761   index_check_without_pop(array, index);
 762 }
 763 
 764 void TemplateTable::index_check_without_pop(Register array, Register index) {
 765   // destroys rbx
 766   // sign extend index for use by indexed load
 767   __ movl2ptr(index, index);
 768   // check index
 769   __ cmpl(index, Address(array, arrayOopDesc::length_offset_in_bytes()));
 770   if (index != rbx) {
 771     // ??? convention: move aberrant index into rbx for exception message
 772     assert(rbx != array, "different registers");
 773     __ movl(rbx, index);
 774   }
 775   Label skip;
 776   __ jccb(Assembler::below, skip);
 777   // Pass array to create more detailed exceptions.
 778   __ mov(NOT_LP64(rax) LP64_ONLY(c_rarg1), array);
 779   __ jump(RuntimeAddress(Interpreter::_throw_ArrayIndexOutOfBoundsException_entry));
 780   __ bind(skip);
 781 }
 782 
 783 void TemplateTable::iaload() {
 784   transition(itos, itos);
 785   // rax: index
 786   // rdx: array
 787   index_check(rdx, rax); // kills rbx
 788   __ access_load_at(T_INT, IN_HEAP | IS_ARRAY, rax,
 789                     Address(rdx, rax, Address::times_4,
 790                             arrayOopDesc::base_offset_in_bytes(T_INT)),
 791                     noreg, noreg);
 792 }
 793 
 794 void TemplateTable::laload() {
 795   transition(itos, ltos);
 796   // rax: index
 797   // rdx: array
 798   index_check(rdx, rax); // kills rbx
 799   NOT_LP64(__ mov(rbx, rax));
 800   // rbx,: index
 801   __ access_load_at(T_LONG, IN_HEAP | IS_ARRAY, noreg /* ltos */,
 802                     Address(rdx, rbx, Address::times_8,
 803                             arrayOopDesc::base_offset_in_bytes(T_LONG)),
 804                     noreg, noreg);
 805 }
 806 
 807 
 808 
 809 void TemplateTable::faload() {
 810   transition(itos, ftos);
 811   // rax: index
 812   // rdx: array
 813   index_check(rdx, rax); // kills rbx
 814   __ access_load_at(T_FLOAT, IN_HEAP | IS_ARRAY, noreg /* ftos */,
 815                     Address(rdx, rax,
 816                             Address::times_4,
 817                             arrayOopDesc::base_offset_in_bytes(T_FLOAT)),
 818                     noreg, noreg);
 819 }
 820 
 821 void TemplateTable::daload() {
 822   transition(itos, dtos);
 823   // rax: index
 824   // rdx: array
 825   index_check(rdx, rax); // kills rbx
 826   __ access_load_at(T_DOUBLE, IN_HEAP | IS_ARRAY, noreg /* dtos */,
 827                     Address(rdx, rax,
 828                             Address::times_8,
 829                             arrayOopDesc::base_offset_in_bytes(T_DOUBLE)),
 830                     noreg, noreg);
 831 }
 832 
 833 void TemplateTable::aaload() {
 834   transition(itos, atos);
 835   Register array = rdx;
 836   Register index = rax;
 837 
 838   index_check(array, index); // kills rbx
 839   __ profile_array_type<ArrayLoadData>(rbx, array, rcx);
 840   if (UseFlatArray) {
 841     Label is_flat_array, done;
 842     __ test_flat_array_oop(array, rbx, is_flat_array);
 843     do_oop_load(_masm,
 844                 Address(array, index,
 845                         UseCompressedOops ? Address::times_4 : Address::times_ptr,
 846                         arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
 847                 rax,
 848                 IS_ARRAY);
 849     __ jmp(done);
 850     __ bind(is_flat_array);
 851     __ read_flat_element(array, index, rbx, rcx, rax);
 852     __ bind(done);
 853   } else {
 854     do_oop_load(_masm,
 855                 Address(array, index,
 856                         UseCompressedOops ? Address::times_4 : Address::times_ptr,
 857                         arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
 858                 rax,
 859                 IS_ARRAY);
 860   }
 861   __ profile_element_type(rbx, rax, rcx);
 862 }
 863 
 864 void TemplateTable::baload() {
 865   transition(itos, itos);
 866   // rax: index
 867   // rdx: array
 868   index_check(rdx, rax); // kills rbx
 869   __ access_load_at(T_BYTE, IN_HEAP | IS_ARRAY, rax,
 870                     Address(rdx, rax, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_BYTE)),
 871                     noreg, noreg);
 872 }
 873 
 874 void TemplateTable::caload() {
 875   transition(itos, itos);
 876   // rax: index
 877   // rdx: array
 878   index_check(rdx, rax); // kills rbx
 879   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, rax,
 880                     Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)),
 881                     noreg, noreg);
 882 }
 883 
 884 // iload followed by caload frequent pair
 885 void TemplateTable::fast_icaload() {
 886   transition(vtos, itos);
 887   // load index out of locals
 888   locals_index(rbx);
 889   __ movl(rax, iaddress(rbx));
 890 
 891   // rax: index
 892   // rdx: array
 893   index_check(rdx, rax); // kills rbx
 894   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, rax,
 895                     Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)),
 896                     noreg, noreg);
 897 }
 898 
 899 
 900 void TemplateTable::saload() {
 901   transition(itos, itos);
 902   // rax: index
 903   // rdx: array
 904   index_check(rdx, rax); // kills rbx
 905   __ access_load_at(T_SHORT, IN_HEAP | IS_ARRAY, rax,
 906                     Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_SHORT)),
 907                     noreg, noreg);
 908 }
 909 
 910 void TemplateTable::iload(int n) {
 911   transition(vtos, itos);
 912   __ movl(rax, iaddress(n));
 913 }
 914 
 915 void TemplateTable::lload(int n) {
 916   transition(vtos, ltos);
 917   __ movptr(rax, laddress(n));
 918   NOT_LP64(__ movptr(rdx, haddress(n)));
 919 }
 920 
 921 void TemplateTable::fload(int n) {
 922   transition(vtos, ftos);
 923   __ load_float(faddress(n));
 924 }
 925 
 926 void TemplateTable::dload(int n) {
 927   transition(vtos, dtos);
 928   __ load_double(daddress(n));
 929 }
 930 
 931 void TemplateTable::aload(int n) {
 932   transition(vtos, atos);
 933   __ movptr(rax, aaddress(n));
 934 }
 935 
 936 void TemplateTable::aload_0() {
 937   aload_0_internal();
 938 }
 939 
 940 void TemplateTable::nofast_aload_0() {
 941   aload_0_internal(may_not_rewrite);
 942 }
 943 
 944 void TemplateTable::aload_0_internal(RewriteControl rc) {
 945   transition(vtos, atos);
 946   // According to bytecode histograms, the pairs:
 947   //
 948   // _aload_0, _fast_igetfield
 949   // _aload_0, _fast_agetfield
 950   // _aload_0, _fast_fgetfield
 951   //
 952   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 953   // _aload_0 bytecode checks if the next bytecode is either
 954   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 955   // rewrites the current bytecode into a pair bytecode; otherwise it
 956   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 957   // the pair check anymore.
 958   //
 959   // Note: If the next bytecode is _getfield, the rewrite must be
 960   //       delayed, otherwise we may miss an opportunity for a pair.
 961   //
 962   // Also rewrite frequent pairs
 963   //   aload_0, aload_1
 964   //   aload_0, iload_1
 965   // These bytecodes with a small amount of code are most profitable
 966   // to rewrite
 967   if (RewriteFrequentPairs && rc == may_rewrite) {
 968     Label rewrite, done;
 969 
 970     const Register bc = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
 971     LP64_ONLY(assert(rbx != bc, "register damaged"));
 972 
 973     // get next byte
 974     __ load_unsigned_byte(rbx, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 975 
 976     // if _getfield then wait with rewrite
 977     __ cmpl(rbx, Bytecodes::_getfield);
 978     __ jcc(Assembler::equal, done);
 979 
 980     // if _igetfield then rewrite to _fast_iaccess_0
 981     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 982     __ cmpl(rbx, Bytecodes::_fast_igetfield);
 983     __ movl(bc, Bytecodes::_fast_iaccess_0);
 984     __ jccb(Assembler::equal, rewrite);
 985 
 986     // if _agetfield then rewrite to _fast_aaccess_0
 987     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 988     __ cmpl(rbx, Bytecodes::_fast_agetfield);
 989     __ movl(bc, Bytecodes::_fast_aaccess_0);
 990     __ jccb(Assembler::equal, rewrite);
 991 
 992     // if _fgetfield then rewrite to _fast_faccess_0
 993     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 994     __ cmpl(rbx, Bytecodes::_fast_fgetfield);
 995     __ movl(bc, Bytecodes::_fast_faccess_0);
 996     __ jccb(Assembler::equal, rewrite);
 997 
 998     // else rewrite to _fast_aload0
 999     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
1000     __ movl(bc, Bytecodes::_fast_aload_0);
1001 
1002     // rewrite
1003     // bc: fast bytecode
1004     __ bind(rewrite);
1005     patch_bytecode(Bytecodes::_aload_0, bc, rbx, false);
1006 
1007     __ bind(done);
1008   }
1009 
1010   // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
1011   aload(0);
1012 }
1013 
1014 void TemplateTable::istore() {
1015   transition(itos, vtos);
1016   locals_index(rbx);
1017   __ movl(iaddress(rbx), rax);
1018 }
1019 
1020 
1021 void TemplateTable::lstore() {
1022   transition(ltos, vtos);
1023   locals_index(rbx);
1024   __ movptr(laddress(rbx), rax);
1025   NOT_LP64(__ movptr(haddress(rbx), rdx));
1026 }
1027 
1028 void TemplateTable::fstore() {
1029   transition(ftos, vtos);
1030   locals_index(rbx);
1031   __ store_float(faddress(rbx));
1032 }
1033 
1034 void TemplateTable::dstore() {
1035   transition(dtos, vtos);
1036   locals_index(rbx);
1037   __ store_double(daddress(rbx));
1038 }
1039 
1040 void TemplateTable::astore() {
1041   transition(vtos, vtos);
1042   __ pop_ptr(rax);
1043   locals_index(rbx);
1044   __ movptr(aaddress(rbx), rax);
1045 }
1046 
1047 void TemplateTable::wide_istore() {
1048   transition(vtos, vtos);
1049   __ pop_i();
1050   locals_index_wide(rbx);
1051   __ movl(iaddress(rbx), rax);
1052 }
1053 
1054 void TemplateTable::wide_lstore() {
1055   transition(vtos, vtos);
1056   NOT_LP64(__ pop_l(rax, rdx));
1057   LP64_ONLY(__ pop_l());
1058   locals_index_wide(rbx);
1059   __ movptr(laddress(rbx), rax);
1060   NOT_LP64(__ movl(haddress(rbx), rdx));
1061 }
1062 
1063 void TemplateTable::wide_fstore() {
1064 #ifdef _LP64
1065   transition(vtos, vtos);
1066   __ pop_f(xmm0);
1067   locals_index_wide(rbx);
1068   __ movflt(faddress(rbx), xmm0);
1069 #else
1070   wide_istore();
1071 #endif
1072 }
1073 
1074 void TemplateTable::wide_dstore() {
1075 #ifdef _LP64
1076   transition(vtos, vtos);
1077   __ pop_d(xmm0);
1078   locals_index_wide(rbx);
1079   __ movdbl(daddress(rbx), xmm0);
1080 #else
1081   wide_lstore();
1082 #endif
1083 }
1084 
1085 void TemplateTable::wide_astore() {
1086   transition(vtos, vtos);
1087   __ pop_ptr(rax);
1088   locals_index_wide(rbx);
1089   __ movptr(aaddress(rbx), rax);
1090 }
1091 
1092 void TemplateTable::iastore() {
1093   transition(itos, vtos);
1094   __ pop_i(rbx);
1095   // rax: value
1096   // rbx: index
1097   // rdx: array
1098   index_check(rdx, rbx); // prefer index in rbx
1099   __ access_store_at(T_INT, IN_HEAP | IS_ARRAY,
1100                      Address(rdx, rbx, Address::times_4,
1101                              arrayOopDesc::base_offset_in_bytes(T_INT)),
1102                      rax, noreg, noreg, noreg);
1103 }
1104 
1105 void TemplateTable::lastore() {
1106   transition(ltos, vtos);
1107   __ pop_i(rbx);
1108   // rax,: low(value)
1109   // rcx: array
1110   // rdx: high(value)
1111   index_check(rcx, rbx);  // prefer index in rbx,
1112   // rbx,: index
1113   __ access_store_at(T_LONG, IN_HEAP | IS_ARRAY,
1114                      Address(rcx, rbx, Address::times_8,
1115                              arrayOopDesc::base_offset_in_bytes(T_LONG)),
1116                      noreg /* ltos */, noreg, noreg, noreg);
1117 }
1118 
1119 
1120 void TemplateTable::fastore() {
1121   transition(ftos, vtos);
1122   __ pop_i(rbx);
1123   // value is in UseSSE >= 1 ? xmm0 : ST(0)
1124   // rbx:  index
1125   // rdx:  array
1126   index_check(rdx, rbx); // prefer index in rbx
1127   __ access_store_at(T_FLOAT, IN_HEAP | IS_ARRAY,
1128                      Address(rdx, rbx, Address::times_4,
1129                              arrayOopDesc::base_offset_in_bytes(T_FLOAT)),
1130                      noreg /* ftos */, noreg, noreg, noreg);
1131 }
1132 
1133 void TemplateTable::dastore() {
1134   transition(dtos, vtos);
1135   __ pop_i(rbx);
1136   // value is in UseSSE >= 2 ? xmm0 : ST(0)
1137   // rbx:  index
1138   // rdx:  array
1139   index_check(rdx, rbx); // prefer index in rbx
1140   __ access_store_at(T_DOUBLE, IN_HEAP | IS_ARRAY,
1141                      Address(rdx, rbx, Address::times_8,
1142                              arrayOopDesc::base_offset_in_bytes(T_DOUBLE)),
1143                      noreg /* dtos */, noreg, noreg, noreg);
1144 }
1145 
1146 void TemplateTable::aastore() {
1147   Label is_null, is_flat_array, ok_is_subtype, done;
1148   transition(vtos, vtos);
1149   // stack: ..., array, index, value
1150   __ movptr(rax, at_tos());    // value
1151   __ movl(rcx, at_tos_p1()); // index
1152   __ movptr(rdx, at_tos_p2()); // array
1153 
1154   Address element_address(rdx, rcx,
1155                           UseCompressedOops? Address::times_4 : Address::times_ptr,
1156                           arrayOopDesc::base_offset_in_bytes(T_OBJECT));
1157 
1158   index_check_without_pop(rdx, rcx);     // kills rbx
1159 
1160   __ profile_array_type<ArrayStoreData>(rdi, rdx, rbx);
1161   __ profile_multiple_element_types(rdi, rax, rbx, rcx);
1162 
1163   __ testptr(rax, rax);
1164   __ jcc(Assembler::zero, is_null);
1165 
1166   // Move array class to rdi
1167   __ load_klass(rdi, rdx, rscratch1);
1168   if (UseFlatArray) {
1169     __ movl(rbx, Address(rdi, Klass::layout_helper_offset()));
1170     __ test_flat_array_layout(rbx, is_flat_array);
1171   }
1172 
1173   // Move subklass into rbx
1174   __ load_klass(rbx, rax, rscratch1);
1175   // Move array element superklass into rax
1176   __ movptr(rax, Address(rdi,
1177                          ObjArrayKlass::element_klass_offset()));
1178 
1179   // Generate subtype check.  Blows rcx, rdi
1180   // Superklass in rax.  Subklass in rbx.
1181   // is "rbx <: rax" ? (value subclass <: array element superclass)
1182   __ gen_subtype_check(rbx, ok_is_subtype, false);
1183 
1184   // Come here on failure
1185   // object is at TOS
1186   __ jump(RuntimeAddress(Interpreter::_throw_ArrayStoreException_entry));
1187 
1188   // Come here on success
1189   __ bind(ok_is_subtype);
1190 
1191   // Get the value we will store
1192   __ movptr(rax, at_tos());
1193   __ movl(rcx, at_tos_p1()); // index
1194   // Now store using the appropriate barrier
1195   do_oop_store(_masm, element_address, rax, IS_ARRAY);
1196   __ jmp(done);
1197 
1198   // Have a null in rax, rdx=array, ecx=index.  Store null at ary[idx]
1199   __ bind(is_null);
1200   if (EnableValhalla) {
1201     Label is_null_into_value_array_npe, store_null;
1202 
1203     // No way to store null in null-free array
1204     __ test_null_free_array_oop(rdx, rbx, is_null_into_value_array_npe);
1205     __ jmp(store_null);
1206 
1207     __ bind(is_null_into_value_array_npe);
1208     __ jump(RuntimeAddress(Interpreter::_throw_NullPointerException_entry));
1209 
1210     __ bind(store_null);
1211   }
1212   // Store a null
1213   do_oop_store(_masm, element_address, noreg, IS_ARRAY);
1214   __ jmp(done);
1215 
1216   if (UseFlatArray) {
1217     Label is_type_ok;
1218     __ bind(is_flat_array); // Store non-null value to flat
1219 
1220     // Simplistic type check...
1221 
1222     // Profile the not-null value's klass.
1223     __ load_klass(rbx, rax, rscratch1);
1224     // Move element klass into rax
1225     __ movptr(rax, Address(rdi, ArrayKlass::element_klass_offset()));
1226     // flat value array needs exact type match
1227     // is "rax == rbx" (value subclass == array element superclass)
1228     __ cmpptr(rax, rbx);
1229     __ jccb(Assembler::equal, is_type_ok);
1230 
1231     __ jump(RuntimeAddress(Interpreter::_throw_ArrayStoreException_entry));
1232 
1233     __ bind(is_type_ok);
1234     // rbx: value's klass
1235     // rdx: array
1236     // rdi: array klass
1237     __ test_klass_is_empty_inline_type(rbx, rax, done);
1238 
1239     // calc dst for copy
1240     __ movl(rax, at_tos_p1()); // index
1241     __ data_for_value_array_index(rdx, rdi, rax, rax);
1242 
1243     // ...and src for copy
1244     __ movptr(rcx, at_tos());  // value
1245     __ data_for_oop(rcx, rcx, rbx);
1246 
1247     __ access_value_copy(IN_HEAP, rcx, rax, rbx);
1248   }
1249   // Pop stack arguments
1250   __ bind(done);
1251   __ addptr(rsp, 3 * Interpreter::stackElementSize);
1252 }
1253 
1254 void TemplateTable::bastore() {
1255   transition(itos, vtos);
1256   __ pop_i(rbx);
1257   // rax: value
1258   // rbx: index
1259   // rdx: array
1260   index_check(rdx, rbx); // prefer index in rbx
1261   // Need to check whether array is boolean or byte
1262   // since both types share the bastore bytecode.
1263   __ load_klass(rcx, rdx, rscratch1);
1264   __ movl(rcx, Address(rcx, Klass::layout_helper_offset()));
1265   int diffbit = Klass::layout_helper_boolean_diffbit();
1266   __ testl(rcx, diffbit);
1267   Label L_skip;
1268   __ jccb(Assembler::zero, L_skip);
1269   __ andl(rax, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1270   __ bind(L_skip);
1271   __ access_store_at(T_BYTE, IN_HEAP | IS_ARRAY,
1272                      Address(rdx, rbx,Address::times_1,
1273                              arrayOopDesc::base_offset_in_bytes(T_BYTE)),
1274                      rax, noreg, noreg, noreg);
1275 }
1276 
1277 void TemplateTable::castore() {
1278   transition(itos, vtos);
1279   __ pop_i(rbx);
1280   // rax: value
1281   // rbx: index
1282   // rdx: array
1283   index_check(rdx, rbx);  // prefer index in rbx
1284   __ access_store_at(T_CHAR, IN_HEAP | IS_ARRAY,
1285                      Address(rdx, rbx, Address::times_2,
1286                              arrayOopDesc::base_offset_in_bytes(T_CHAR)),
1287                      rax, noreg, noreg, noreg);
1288 }
1289 
1290 
1291 void TemplateTable::sastore() {
1292   castore();
1293 }
1294 
1295 void TemplateTable::istore(int n) {
1296   transition(itos, vtos);
1297   __ movl(iaddress(n), rax);
1298 }
1299 
1300 void TemplateTable::lstore(int n) {
1301   transition(ltos, vtos);
1302   __ movptr(laddress(n), rax);
1303   NOT_LP64(__ movptr(haddress(n), rdx));
1304 }
1305 
1306 void TemplateTable::fstore(int n) {
1307   transition(ftos, vtos);
1308   __ store_float(faddress(n));
1309 }
1310 
1311 void TemplateTable::dstore(int n) {
1312   transition(dtos, vtos);
1313   __ store_double(daddress(n));
1314 }
1315 
1316 
1317 void TemplateTable::astore(int n) {
1318   transition(vtos, vtos);
1319   __ pop_ptr(rax);
1320   __ movptr(aaddress(n), rax);
1321 }
1322 
1323 void TemplateTable::pop() {
1324   transition(vtos, vtos);
1325   __ addptr(rsp, Interpreter::stackElementSize);
1326 }
1327 
1328 void TemplateTable::pop2() {
1329   transition(vtos, vtos);
1330   __ addptr(rsp, 2 * Interpreter::stackElementSize);
1331 }
1332 
1333 
1334 void TemplateTable::dup() {
1335   transition(vtos, vtos);
1336   __ load_ptr(0, rax);
1337   __ push_ptr(rax);
1338   // stack: ..., a, a
1339 }
1340 
1341 void TemplateTable::dup_x1() {
1342   transition(vtos, vtos);
1343   // stack: ..., a, b
1344   __ load_ptr( 0, rax);  // load b
1345   __ load_ptr( 1, rcx);  // load a
1346   __ store_ptr(1, rax);  // store b
1347   __ store_ptr(0, rcx);  // store a
1348   __ push_ptr(rax);      // push b
1349   // stack: ..., b, a, b
1350 }
1351 
1352 void TemplateTable::dup_x2() {
1353   transition(vtos, vtos);
1354   // stack: ..., a, b, c
1355   __ load_ptr( 0, rax);  // load c
1356   __ load_ptr( 2, rcx);  // load a
1357   __ store_ptr(2, rax);  // store c in a
1358   __ push_ptr(rax);      // push c
1359   // stack: ..., c, b, c, c
1360   __ load_ptr( 2, rax);  // load b
1361   __ store_ptr(2, rcx);  // store a in b
1362   // stack: ..., c, a, c, c
1363   __ store_ptr(1, rax);  // store b in c
1364   // stack: ..., c, a, b, c
1365 }
1366 
1367 void TemplateTable::dup2() {
1368   transition(vtos, vtos);
1369   // stack: ..., a, b
1370   __ load_ptr(1, rax);  // load a
1371   __ push_ptr(rax);     // push a
1372   __ load_ptr(1, rax);  // load b
1373   __ push_ptr(rax);     // push b
1374   // stack: ..., a, b, a, b
1375 }
1376 
1377 
1378 void TemplateTable::dup2_x1() {
1379   transition(vtos, vtos);
1380   // stack: ..., a, b, c
1381   __ load_ptr( 0, rcx);  // load c
1382   __ load_ptr( 1, rax);  // load b
1383   __ push_ptr(rax);      // push b
1384   __ push_ptr(rcx);      // push c
1385   // stack: ..., a, b, c, b, c
1386   __ store_ptr(3, rcx);  // store c in b
1387   // stack: ..., a, c, c, b, c
1388   __ load_ptr( 4, rcx);  // load a
1389   __ store_ptr(2, rcx);  // store a in 2nd c
1390   // stack: ..., a, c, a, b, c
1391   __ store_ptr(4, rax);  // store b in a
1392   // stack: ..., b, c, a, b, c
1393 }
1394 
1395 void TemplateTable::dup2_x2() {
1396   transition(vtos, vtos);
1397   // stack: ..., a, b, c, d
1398   __ load_ptr( 0, rcx);  // load d
1399   __ load_ptr( 1, rax);  // load c
1400   __ push_ptr(rax);      // push c
1401   __ push_ptr(rcx);      // push d
1402   // stack: ..., a, b, c, d, c, d
1403   __ load_ptr( 4, rax);  // load b
1404   __ store_ptr(2, rax);  // store b in d
1405   __ store_ptr(4, rcx);  // store d in b
1406   // stack: ..., a, d, c, b, c, d
1407   __ load_ptr( 5, rcx);  // load a
1408   __ load_ptr( 3, rax);  // load c
1409   __ store_ptr(3, rcx);  // store a in c
1410   __ store_ptr(5, rax);  // store c in a
1411   // stack: ..., c, d, a, b, c, d
1412 }
1413 
1414 void TemplateTable::swap() {
1415   transition(vtos, vtos);
1416   // stack: ..., a, b
1417   __ load_ptr( 1, rcx);  // load a
1418   __ load_ptr( 0, rax);  // load b
1419   __ store_ptr(0, rcx);  // store a in b
1420   __ store_ptr(1, rax);  // store b in a
1421   // stack: ..., b, a
1422 }
1423 
1424 void TemplateTable::iop2(Operation op) {
1425   transition(itos, itos);
1426   switch (op) {
1427   case add  :                    __ pop_i(rdx); __ addl (rax, rdx); break;
1428   case sub  : __ movl(rdx, rax); __ pop_i(rax); __ subl (rax, rdx); break;
1429   case mul  :                    __ pop_i(rdx); __ imull(rax, rdx); break;
1430   case _and :                    __ pop_i(rdx); __ andl (rax, rdx); break;
1431   case _or  :                    __ pop_i(rdx); __ orl  (rax, rdx); break;
1432   case _xor :                    __ pop_i(rdx); __ xorl (rax, rdx); break;
1433   case shl  : __ movl(rcx, rax); __ pop_i(rax); __ shll (rax);      break;
1434   case shr  : __ movl(rcx, rax); __ pop_i(rax); __ sarl (rax);      break;
1435   case ushr : __ movl(rcx, rax); __ pop_i(rax); __ shrl (rax);      break;
1436   default   : ShouldNotReachHere();
1437   }
1438 }
1439 
1440 void TemplateTable::lop2(Operation op) {
1441   transition(ltos, ltos);
1442 #ifdef _LP64
1443   switch (op) {
1444   case add  :                    __ pop_l(rdx); __ addptr(rax, rdx); break;
1445   case sub  : __ mov(rdx, rax);  __ pop_l(rax); __ subptr(rax, rdx); break;
1446   case _and :                    __ pop_l(rdx); __ andptr(rax, rdx); break;
1447   case _or  :                    __ pop_l(rdx); __ orptr (rax, rdx); break;
1448   case _xor :                    __ pop_l(rdx); __ xorptr(rax, rdx); break;
1449   default   : ShouldNotReachHere();
1450   }
1451 #else
1452   __ pop_l(rbx, rcx);
1453   switch (op) {
1454     case add  : __ addl(rax, rbx); __ adcl(rdx, rcx); break;
1455     case sub  : __ subl(rbx, rax); __ sbbl(rcx, rdx);
1456                 __ mov (rax, rbx); __ mov (rdx, rcx); break;
1457     case _and : __ andl(rax, rbx); __ andl(rdx, rcx); break;
1458     case _or  : __ orl (rax, rbx); __ orl (rdx, rcx); break;
1459     case _xor : __ xorl(rax, rbx); __ xorl(rdx, rcx); break;
1460     default   : ShouldNotReachHere();
1461   }
1462 #endif
1463 }
1464 
1465 void TemplateTable::idiv() {
1466   transition(itos, itos);
1467   __ movl(rcx, rax);
1468   __ pop_i(rax);
1469   // Note: could xor rax and ecx and compare with (-1 ^ min_int). If
1470   //       they are not equal, one could do a normal division (no correction
1471   //       needed), which may speed up this implementation for the common case.
1472   //       (see also JVM spec., p.243 & p.271)
1473   __ corrected_idivl(rcx);
1474 }
1475 
1476 void TemplateTable::irem() {
1477   transition(itos, itos);
1478   __ movl(rcx, rax);
1479   __ pop_i(rax);
1480   // Note: could xor rax and ecx and compare with (-1 ^ min_int). If
1481   //       they are not equal, one could do a normal division (no correction
1482   //       needed), which may speed up this implementation for the common case.
1483   //       (see also JVM spec., p.243 & p.271)
1484   __ corrected_idivl(rcx);
1485   __ movl(rax, rdx);
1486 }
1487 
1488 void TemplateTable::lmul() {
1489   transition(ltos, ltos);
1490 #ifdef _LP64
1491   __ pop_l(rdx);
1492   __ imulq(rax, rdx);
1493 #else
1494   __ pop_l(rbx, rcx);
1495   __ push(rcx); __ push(rbx);
1496   __ push(rdx); __ push(rax);
1497   __ lmul(2 * wordSize, 0);
1498   __ addptr(rsp, 4 * wordSize);  // take off temporaries
1499 #endif
1500 }
1501 
1502 void TemplateTable::ldiv() {
1503   transition(ltos, ltos);
1504 #ifdef _LP64
1505   __ mov(rcx, rax);
1506   __ pop_l(rax);
1507   // generate explicit div0 check
1508   __ testq(rcx, rcx);
1509   __ jump_cc(Assembler::zero,
1510              RuntimeAddress(Interpreter::_throw_ArithmeticException_entry));
1511   // Note: could xor rax and rcx and compare with (-1 ^ min_int). If
1512   //       they are not equal, one could do a normal division (no correction
1513   //       needed), which may speed up this implementation for the common case.
1514   //       (see also JVM spec., p.243 & p.271)
1515   __ corrected_idivq(rcx); // kills rbx
1516 #else
1517   __ pop_l(rbx, rcx);
1518   __ push(rcx); __ push(rbx);
1519   __ push(rdx); __ push(rax);
1520   // check if y = 0
1521   __ orl(rax, rdx);
1522   __ jump_cc(Assembler::zero,
1523              RuntimeAddress(Interpreter::_throw_ArithmeticException_entry));
1524   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::ldiv));
1525   __ addptr(rsp, 4 * wordSize);  // take off temporaries
1526 #endif
1527 }
1528 
1529 void TemplateTable::lrem() {
1530   transition(ltos, ltos);
1531 #ifdef _LP64
1532   __ mov(rcx, rax);
1533   __ pop_l(rax);
1534   __ testq(rcx, rcx);
1535   __ jump_cc(Assembler::zero,
1536              RuntimeAddress(Interpreter::_throw_ArithmeticException_entry));
1537   // Note: could xor rax and rcx and compare with (-1 ^ min_int). If
1538   //       they are not equal, one could do a normal division (no correction
1539   //       needed), which may speed up this implementation for the common case.
1540   //       (see also JVM spec., p.243 & p.271)
1541   __ corrected_idivq(rcx); // kills rbx
1542   __ mov(rax, rdx);
1543 #else
1544   __ pop_l(rbx, rcx);
1545   __ push(rcx); __ push(rbx);
1546   __ push(rdx); __ push(rax);
1547   // check if y = 0
1548   __ orl(rax, rdx);
1549   __ jump_cc(Assembler::zero,
1550              RuntimeAddress(Interpreter::_throw_ArithmeticException_entry));
1551   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::lrem));
1552   __ addptr(rsp, 4 * wordSize);
1553 #endif
1554 }
1555 
1556 void TemplateTable::lshl() {
1557   transition(itos, ltos);
1558   __ movl(rcx, rax);                             // get shift count
1559   #ifdef _LP64
1560   __ pop_l(rax);                                 // get shift value
1561   __ shlq(rax);
1562 #else
1563   __ pop_l(rax, rdx);                            // get shift value
1564   __ lshl(rdx, rax);
1565 #endif
1566 }
1567 
1568 void TemplateTable::lshr() {
1569 #ifdef _LP64
1570   transition(itos, ltos);
1571   __ movl(rcx, rax);                             // get shift count
1572   __ pop_l(rax);                                 // get shift value
1573   __ sarq(rax);
1574 #else
1575   transition(itos, ltos);
1576   __ mov(rcx, rax);                              // get shift count
1577   __ pop_l(rax, rdx);                            // get shift value
1578   __ lshr(rdx, rax, true);
1579 #endif
1580 }
1581 
1582 void TemplateTable::lushr() {
1583   transition(itos, ltos);
1584 #ifdef _LP64
1585   __ movl(rcx, rax);                             // get shift count
1586   __ pop_l(rax);                                 // get shift value
1587   __ shrq(rax);
1588 #else
1589   __ mov(rcx, rax);                              // get shift count
1590   __ pop_l(rax, rdx);                            // get shift value
1591   __ lshr(rdx, rax);
1592 #endif
1593 }
1594 
1595 void TemplateTable::fop2(Operation op) {
1596   transition(ftos, ftos);
1597 
1598   if (UseSSE >= 1) {
1599     switch (op) {
1600     case add:
1601       __ addss(xmm0, at_rsp());
1602       __ addptr(rsp, Interpreter::stackElementSize);
1603       break;
1604     case sub:
1605       __ movflt(xmm1, xmm0);
1606       __ pop_f(xmm0);
1607       __ subss(xmm0, xmm1);
1608       break;
1609     case mul:
1610       __ mulss(xmm0, at_rsp());
1611       __ addptr(rsp, Interpreter::stackElementSize);
1612       break;
1613     case div:
1614       __ movflt(xmm1, xmm0);
1615       __ pop_f(xmm0);
1616       __ divss(xmm0, xmm1);
1617       break;
1618     case rem:
1619       // On x86_64 platforms the SharedRuntime::frem method is called to perform the
1620       // modulo operation. The frem method calls the function
1621       // double fmod(double x, double y) in math.h. The documentation of fmod states:
1622       // "If x or y is a NaN, a NaN is returned." without specifying what type of NaN
1623       // (signalling or quiet) is returned.
1624       //
1625       // On x86_32 platforms the FPU is used to perform the modulo operation. The
1626       // reason is that on 32-bit Windows the sign of modulo operations diverges from
1627       // what is considered the standard (e.g., -0.0f % -3.14f is 0.0f (and not -0.0f).
1628       // The fprem instruction used on x86_32 is functionally equivalent to
1629       // SharedRuntime::frem in that it returns a NaN.
1630 #ifdef _LP64
1631       __ movflt(xmm1, xmm0);
1632       __ pop_f(xmm0);
1633       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem), 2);
1634 #else // !_LP64
1635       __ push_f(xmm0);
1636       __ pop_f();
1637       __ fld_s(at_rsp());
1638       __ fremr(rax);
1639       __ f2ieee();
1640       __ pop(rax);  // pop second operand off the stack
1641       __ push_f();
1642       __ pop_f(xmm0);
1643 #endif // _LP64
1644       break;
1645     default:
1646       ShouldNotReachHere();
1647       break;
1648     }
1649   } else {
1650 #ifdef _LP64
1651     ShouldNotReachHere();
1652 #else // !_LP64
1653     switch (op) {
1654     case add: __ fadd_s (at_rsp());                break;
1655     case sub: __ fsubr_s(at_rsp());                break;
1656     case mul: __ fmul_s (at_rsp());                break;
1657     case div: __ fdivr_s(at_rsp());                break;
1658     case rem: __ fld_s  (at_rsp()); __ fremr(rax); break;
1659     default : ShouldNotReachHere();
1660     }
1661     __ f2ieee();
1662     __ pop(rax);  // pop second operand off the stack
1663 #endif // _LP64
1664   }
1665 }
1666 
1667 void TemplateTable::dop2(Operation op) {
1668   transition(dtos, dtos);
1669   if (UseSSE >= 2) {
1670     switch (op) {
1671     case add:
1672       __ addsd(xmm0, at_rsp());
1673       __ addptr(rsp, 2 * Interpreter::stackElementSize);
1674       break;
1675     case sub:
1676       __ movdbl(xmm1, xmm0);
1677       __ pop_d(xmm0);
1678       __ subsd(xmm0, xmm1);
1679       break;
1680     case mul:
1681       __ mulsd(xmm0, at_rsp());
1682       __ addptr(rsp, 2 * Interpreter::stackElementSize);
1683       break;
1684     case div:
1685       __ movdbl(xmm1, xmm0);
1686       __ pop_d(xmm0);
1687       __ divsd(xmm0, xmm1);
1688       break;
1689     case rem:
1690       // Similar to fop2(), the modulo operation is performed using the
1691       // SharedRuntime::drem method (on x86_64 platforms) or using the
1692       // FPU (on x86_32 platforms) for the same reasons as mentioned in fop2().
1693 #ifdef _LP64
1694       __ movdbl(xmm1, xmm0);
1695       __ pop_d(xmm0);
1696       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem), 2);
1697 #else // !_LP64
1698       __ push_d(xmm0);
1699       __ pop_d();
1700       __ fld_d(at_rsp());
1701       __ fremr(rax);
1702       __ d2ieee();
1703       __ pop(rax);
1704       __ pop(rdx);
1705       __ push_d();
1706       __ pop_d(xmm0);
1707 #endif // _LP64
1708       break;
1709     default:
1710       ShouldNotReachHere();
1711       break;
1712     }
1713   } else {
1714 #ifdef _LP64
1715     ShouldNotReachHere();
1716 #else // !_LP64
1717     switch (op) {
1718     case add: __ fadd_d (at_rsp());                break;
1719     case sub: __ fsubr_d(at_rsp());                break;
1720     case mul: {
1721       // strict semantics
1722       __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias1()));
1723       __ fmulp();
1724       __ fmul_d (at_rsp());
1725       __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias2()));
1726       __ fmulp();
1727       break;
1728     }
1729     case div: {
1730       // strict semantics
1731       __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias1()));
1732       __ fmul_d (at_rsp());
1733       __ fdivrp();
1734       __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias2()));
1735       __ fmulp();
1736       break;
1737     }
1738     case rem: __ fld_d  (at_rsp()); __ fremr(rax); break;
1739     default : ShouldNotReachHere();
1740     }
1741     __ d2ieee();
1742     // Pop double precision number from rsp.
1743     __ pop(rax);
1744     __ pop(rdx);
1745 #endif // _LP64
1746   }
1747 }
1748 
1749 void TemplateTable::ineg() {
1750   transition(itos, itos);
1751   __ negl(rax);
1752 }
1753 
1754 void TemplateTable::lneg() {
1755   transition(ltos, ltos);
1756   LP64_ONLY(__ negq(rax));
1757   NOT_LP64(__ lneg(rdx, rax));
1758 }
1759 
1760 // Note: 'double' and 'long long' have 32-bits alignment on x86.
1761 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
1762   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
1763   // of 128-bits operands for SSE instructions.
1764   jlong *operand = (jlong*)(((intptr_t)adr)&((intptr_t)(~0xF)));
1765   // Store the value to a 128-bits operand.
1766   operand[0] = lo;
1767   operand[1] = hi;
1768   return operand;
1769 }
1770 
1771 // Buffer for 128-bits masks used by SSE instructions.
1772 static jlong float_signflip_pool[2*2];
1773 static jlong double_signflip_pool[2*2];
1774 
1775 void TemplateTable::fneg() {
1776   transition(ftos, ftos);
1777   if (UseSSE >= 1) {
1778     static jlong *float_signflip  = double_quadword(&float_signflip_pool[1],  CONST64(0x8000000080000000),  CONST64(0x8000000080000000));
1779     __ xorps(xmm0, ExternalAddress((address) float_signflip), rscratch1);
1780   } else {
1781     LP64_ONLY(ShouldNotReachHere());
1782     NOT_LP64(__ fchs());
1783   }
1784 }
1785 
1786 void TemplateTable::dneg() {
1787   transition(dtos, dtos);
1788   if (UseSSE >= 2) {
1789     static jlong *double_signflip =
1790       double_quadword(&double_signflip_pool[1], CONST64(0x8000000000000000), CONST64(0x8000000000000000));
1791     __ xorpd(xmm0, ExternalAddress((address) double_signflip), rscratch1);
1792   } else {
1793 #ifdef _LP64
1794     ShouldNotReachHere();
1795 #else
1796     __ fchs();
1797 #endif
1798   }
1799 }
1800 
1801 void TemplateTable::iinc() {
1802   transition(vtos, vtos);
1803   __ load_signed_byte(rdx, at_bcp(2)); // get constant
1804   locals_index(rbx);
1805   __ addl(iaddress(rbx), rdx);
1806 }
1807 
1808 void TemplateTable::wide_iinc() {
1809   transition(vtos, vtos);
1810   __ movl(rdx, at_bcp(4)); // get constant
1811   locals_index_wide(rbx);
1812   __ bswapl(rdx); // swap bytes & sign-extend constant
1813   __ sarl(rdx, 16);
1814   __ addl(iaddress(rbx), rdx);
1815   // Note: should probably use only one movl to get both
1816   //       the index and the constant -> fix this
1817 }
1818 
1819 void TemplateTable::convert() {
1820 #ifdef _LP64
1821   // Checking
1822 #ifdef ASSERT
1823   {
1824     TosState tos_in  = ilgl;
1825     TosState tos_out = ilgl;
1826     switch (bytecode()) {
1827     case Bytecodes::_i2l: // fall through
1828     case Bytecodes::_i2f: // fall through
1829     case Bytecodes::_i2d: // fall through
1830     case Bytecodes::_i2b: // fall through
1831     case Bytecodes::_i2c: // fall through
1832     case Bytecodes::_i2s: tos_in = itos; break;
1833     case Bytecodes::_l2i: // fall through
1834     case Bytecodes::_l2f: // fall through
1835     case Bytecodes::_l2d: tos_in = ltos; break;
1836     case Bytecodes::_f2i: // fall through
1837     case Bytecodes::_f2l: // fall through
1838     case Bytecodes::_f2d: tos_in = ftos; break;
1839     case Bytecodes::_d2i: // fall through
1840     case Bytecodes::_d2l: // fall through
1841     case Bytecodes::_d2f: tos_in = dtos; break;
1842     default             : ShouldNotReachHere();
1843     }
1844     switch (bytecode()) {
1845     case Bytecodes::_l2i: // fall through
1846     case Bytecodes::_f2i: // fall through
1847     case Bytecodes::_d2i: // fall through
1848     case Bytecodes::_i2b: // fall through
1849     case Bytecodes::_i2c: // fall through
1850     case Bytecodes::_i2s: tos_out = itos; break;
1851     case Bytecodes::_i2l: // fall through
1852     case Bytecodes::_f2l: // fall through
1853     case Bytecodes::_d2l: tos_out = ltos; break;
1854     case Bytecodes::_i2f: // fall through
1855     case Bytecodes::_l2f: // fall through
1856     case Bytecodes::_d2f: tos_out = ftos; break;
1857     case Bytecodes::_i2d: // fall through
1858     case Bytecodes::_l2d: // fall through
1859     case Bytecodes::_f2d: tos_out = dtos; break;
1860     default             : ShouldNotReachHere();
1861     }
1862     transition(tos_in, tos_out);
1863   }
1864 #endif // ASSERT
1865 
1866   static const int64_t is_nan = 0x8000000000000000L;
1867 
1868   // Conversion
1869   switch (bytecode()) {
1870   case Bytecodes::_i2l:
1871     __ movslq(rax, rax);
1872     break;
1873   case Bytecodes::_i2f:
1874     __ cvtsi2ssl(xmm0, rax);
1875     break;
1876   case Bytecodes::_i2d:
1877     __ cvtsi2sdl(xmm0, rax);
1878     break;
1879   case Bytecodes::_i2b:
1880     __ movsbl(rax, rax);
1881     break;
1882   case Bytecodes::_i2c:
1883     __ movzwl(rax, rax);
1884     break;
1885   case Bytecodes::_i2s:
1886     __ movswl(rax, rax);
1887     break;
1888   case Bytecodes::_l2i:
1889     __ movl(rax, rax);
1890     break;
1891   case Bytecodes::_l2f:
1892     __ cvtsi2ssq(xmm0, rax);
1893     break;
1894   case Bytecodes::_l2d:
1895     __ cvtsi2sdq(xmm0, rax);
1896     break;
1897   case Bytecodes::_f2i:
1898   {
1899     Label L;
1900     __ cvttss2sil(rax, xmm0);
1901     __ cmpl(rax, 0x80000000); // NaN or overflow/underflow?
1902     __ jcc(Assembler::notEqual, L);
1903     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2i), 1);
1904     __ bind(L);
1905   }
1906     break;
1907   case Bytecodes::_f2l:
1908   {
1909     Label L;
1910     __ cvttss2siq(rax, xmm0);
1911     // NaN or overflow/underflow?
1912     __ cmp64(rax, ExternalAddress((address) &is_nan), rscratch1);
1913     __ jcc(Assembler::notEqual, L);
1914     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2l), 1);
1915     __ bind(L);
1916   }
1917     break;
1918   case Bytecodes::_f2d:
1919     __ cvtss2sd(xmm0, xmm0);
1920     break;
1921   case Bytecodes::_d2i:
1922   {
1923     Label L;
1924     __ cvttsd2sil(rax, xmm0);
1925     __ cmpl(rax, 0x80000000); // NaN or overflow/underflow?
1926     __ jcc(Assembler::notEqual, L);
1927     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2i), 1);
1928     __ bind(L);
1929   }
1930     break;
1931   case Bytecodes::_d2l:
1932   {
1933     Label L;
1934     __ cvttsd2siq(rax, xmm0);
1935     // NaN or overflow/underflow?
1936     __ cmp64(rax, ExternalAddress((address) &is_nan), rscratch1);
1937     __ jcc(Assembler::notEqual, L);
1938     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2l), 1);
1939     __ bind(L);
1940   }
1941     break;
1942   case Bytecodes::_d2f:
1943     __ cvtsd2ss(xmm0, xmm0);
1944     break;
1945   default:
1946     ShouldNotReachHere();
1947   }
1948 #else // !_LP64
1949   // Checking
1950 #ifdef ASSERT
1951   { TosState tos_in  = ilgl;
1952     TosState tos_out = ilgl;
1953     switch (bytecode()) {
1954       case Bytecodes::_i2l: // fall through
1955       case Bytecodes::_i2f: // fall through
1956       case Bytecodes::_i2d: // fall through
1957       case Bytecodes::_i2b: // fall through
1958       case Bytecodes::_i2c: // fall through
1959       case Bytecodes::_i2s: tos_in = itos; break;
1960       case Bytecodes::_l2i: // fall through
1961       case Bytecodes::_l2f: // fall through
1962       case Bytecodes::_l2d: tos_in = ltos; break;
1963       case Bytecodes::_f2i: // fall through
1964       case Bytecodes::_f2l: // fall through
1965       case Bytecodes::_f2d: tos_in = ftos; break;
1966       case Bytecodes::_d2i: // fall through
1967       case Bytecodes::_d2l: // fall through
1968       case Bytecodes::_d2f: tos_in = dtos; break;
1969       default             : ShouldNotReachHere();
1970     }
1971     switch (bytecode()) {
1972       case Bytecodes::_l2i: // fall through
1973       case Bytecodes::_f2i: // fall through
1974       case Bytecodes::_d2i: // fall through
1975       case Bytecodes::_i2b: // fall through
1976       case Bytecodes::_i2c: // fall through
1977       case Bytecodes::_i2s: tos_out = itos; break;
1978       case Bytecodes::_i2l: // fall through
1979       case Bytecodes::_f2l: // fall through
1980       case Bytecodes::_d2l: tos_out = ltos; break;
1981       case Bytecodes::_i2f: // fall through
1982       case Bytecodes::_l2f: // fall through
1983       case Bytecodes::_d2f: tos_out = ftos; break;
1984       case Bytecodes::_i2d: // fall through
1985       case Bytecodes::_l2d: // fall through
1986       case Bytecodes::_f2d: tos_out = dtos; break;
1987       default             : ShouldNotReachHere();
1988     }
1989     transition(tos_in, tos_out);
1990   }
1991 #endif // ASSERT
1992 
1993   // Conversion
1994   // (Note: use push(rcx)/pop(rcx) for 1/2-word stack-ptr manipulation)
1995   switch (bytecode()) {
1996     case Bytecodes::_i2l:
1997       __ extend_sign(rdx, rax);
1998       break;
1999     case Bytecodes::_i2f:
2000       if (UseSSE >= 1) {
2001         __ cvtsi2ssl(xmm0, rax);
2002       } else {
2003         __ push(rax);          // store int on tos
2004         __ fild_s(at_rsp());   // load int to ST0
2005         __ f2ieee();           // truncate to float size
2006         __ pop(rcx);           // adjust rsp
2007       }
2008       break;
2009     case Bytecodes::_i2d:
2010       if (UseSSE >= 2) {
2011         __ cvtsi2sdl(xmm0, rax);
2012       } else {
2013       __ push(rax);          // add one slot for d2ieee()
2014       __ push(rax);          // store int on tos
2015       __ fild_s(at_rsp());   // load int to ST0
2016       __ d2ieee();           // truncate to double size
2017       __ pop(rcx);           // adjust rsp
2018       __ pop(rcx);
2019       }
2020       break;
2021     case Bytecodes::_i2b:
2022       __ shll(rax, 24);      // truncate upper 24 bits
2023       __ sarl(rax, 24);      // and sign-extend byte
2024       LP64_ONLY(__ movsbl(rax, rax));
2025       break;
2026     case Bytecodes::_i2c:
2027       __ andl(rax, 0xFFFF);  // truncate upper 16 bits
2028       LP64_ONLY(__ movzwl(rax, rax));
2029       break;
2030     case Bytecodes::_i2s:
2031       __ shll(rax, 16);      // truncate upper 16 bits
2032       __ sarl(rax, 16);      // and sign-extend short
2033       LP64_ONLY(__ movswl(rax, rax));
2034       break;
2035     case Bytecodes::_l2i:
2036       /* nothing to do */
2037       break;
2038     case Bytecodes::_l2f:
2039       // On 64-bit platforms, the cvtsi2ssq instruction is used to convert
2040       // 64-bit long values to floats. On 32-bit platforms it is not possible
2041       // to use that instruction with 64-bit operands, therefore the FPU is
2042       // used to perform the conversion.
2043       __ push(rdx);          // store long on tos
2044       __ push(rax);
2045       __ fild_d(at_rsp());   // load long to ST0
2046       __ f2ieee();           // truncate to float size
2047       __ pop(rcx);           // adjust rsp
2048       __ pop(rcx);
2049       if (UseSSE >= 1) {
2050         __ push_f();
2051         __ pop_f(xmm0);
2052       }
2053       break;
2054     case Bytecodes::_l2d:
2055       // On 32-bit platforms the FPU is used for conversion because on
2056       // 32-bit platforms it is not not possible to use the cvtsi2sdq
2057       // instruction with 64-bit operands.
2058       __ push(rdx);          // store long on tos
2059       __ push(rax);
2060       __ fild_d(at_rsp());   // load long to ST0
2061       __ d2ieee();           // truncate to double size
2062       __ pop(rcx);           // adjust rsp
2063       __ pop(rcx);
2064       if (UseSSE >= 2) {
2065         __ push_d();
2066         __ pop_d(xmm0);
2067       }
2068       break;
2069     case Bytecodes::_f2i:
2070       // SharedRuntime::f2i does not differentiate between sNaNs and qNaNs
2071       // as it returns 0 for any NaN.
2072       if (UseSSE >= 1) {
2073         __ push_f(xmm0);
2074       } else {
2075         __ push(rcx);          // reserve space for argument
2076         __ fstp_s(at_rsp());   // pass float argument on stack
2077       }
2078       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2i), 1);
2079       break;
2080     case Bytecodes::_f2l:
2081       // SharedRuntime::f2l does not differentiate between sNaNs and qNaNs
2082       // as it returns 0 for any NaN.
2083       if (UseSSE >= 1) {
2084        __ push_f(xmm0);
2085       } else {
2086         __ push(rcx);          // reserve space for argument
2087         __ fstp_s(at_rsp());   // pass float argument on stack
2088       }
2089       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2l), 1);
2090       break;
2091     case Bytecodes::_f2d:
2092       if (UseSSE < 1) {
2093         /* nothing to do */
2094       } else if (UseSSE == 1) {
2095         __ push_f(xmm0);
2096         __ pop_f();
2097       } else { // UseSSE >= 2
2098         __ cvtss2sd(xmm0, xmm0);
2099       }
2100       break;
2101     case Bytecodes::_d2i:
2102       if (UseSSE >= 2) {
2103         __ push_d(xmm0);
2104       } else {
2105         __ push(rcx);          // reserve space for argument
2106         __ push(rcx);
2107         __ fstp_d(at_rsp());   // pass double argument on stack
2108       }
2109       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2i), 2);
2110       break;
2111     case Bytecodes::_d2l:
2112       if (UseSSE >= 2) {
2113         __ push_d(xmm0);
2114       } else {
2115         __ push(rcx);          // reserve space for argument
2116         __ push(rcx);
2117         __ fstp_d(at_rsp());   // pass double argument on stack
2118       }
2119       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2l), 2);
2120       break;
2121     case Bytecodes::_d2f:
2122       if (UseSSE <= 1) {
2123         __ push(rcx);          // reserve space for f2ieee()
2124         __ f2ieee();           // truncate to float size
2125         __ pop(rcx);           // adjust rsp
2126         if (UseSSE == 1) {
2127           // The cvtsd2ss instruction is not available if UseSSE==1, therefore
2128           // the conversion is performed using the FPU in this case.
2129           __ push_f();
2130           __ pop_f(xmm0);
2131         }
2132       } else { // UseSSE >= 2
2133         __ cvtsd2ss(xmm0, xmm0);
2134       }
2135       break;
2136     default             :
2137       ShouldNotReachHere();
2138   }
2139 #endif // _LP64
2140 }
2141 
2142 void TemplateTable::lcmp() {
2143   transition(ltos, itos);
2144 #ifdef _LP64
2145   Label done;
2146   __ pop_l(rdx);
2147   __ cmpq(rdx, rax);
2148   __ movl(rax, -1);
2149   __ jccb(Assembler::less, done);
2150   __ setb(Assembler::notEqual, rax);
2151   __ movzbl(rax, rax);
2152   __ bind(done);
2153 #else
2154 
2155   // y = rdx:rax
2156   __ pop_l(rbx, rcx);             // get x = rcx:rbx
2157   __ lcmp2int(rcx, rbx, rdx, rax);// rcx := cmp(x, y)
2158   __ mov(rax, rcx);
2159 #endif
2160 }
2161 
2162 void TemplateTable::float_cmp(bool is_float, int unordered_result) {
2163   if ((is_float && UseSSE >= 1) ||
2164       (!is_float && UseSSE >= 2)) {
2165     Label done;
2166     if (is_float) {
2167       // XXX get rid of pop here, use ... reg, mem32
2168       __ pop_f(xmm1);
2169       __ ucomiss(xmm1, xmm0);
2170     } else {
2171       // XXX get rid of pop here, use ... reg, mem64
2172       __ pop_d(xmm1);
2173       __ ucomisd(xmm1, xmm0);
2174     }
2175     if (unordered_result < 0) {
2176       __ movl(rax, -1);
2177       __ jccb(Assembler::parity, done);
2178       __ jccb(Assembler::below, done);
2179       __ setb(Assembler::notEqual, rdx);
2180       __ movzbl(rax, rdx);
2181     } else {
2182       __ movl(rax, 1);
2183       __ jccb(Assembler::parity, done);
2184       __ jccb(Assembler::above, done);
2185       __ movl(rax, 0);
2186       __ jccb(Assembler::equal, done);
2187       __ decrementl(rax);
2188     }
2189     __ bind(done);
2190   } else {
2191 #ifdef _LP64
2192     ShouldNotReachHere();
2193 #else // !_LP64
2194     if (is_float) {
2195       __ fld_s(at_rsp());
2196     } else {
2197       __ fld_d(at_rsp());
2198       __ pop(rdx);
2199     }
2200     __ pop(rcx);
2201     __ fcmp2int(rax, unordered_result < 0);
2202 #endif // _LP64
2203   }
2204 }
2205 
2206 void TemplateTable::branch(bool is_jsr, bool is_wide) {
2207   __ get_method(rcx); // rcx holds method
2208   __ profile_taken_branch(rax, rbx); // rax holds updated MDP, rbx
2209                                      // holds bumped taken count
2210 
2211   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
2212                              InvocationCounter::counter_offset();
2213   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
2214                               InvocationCounter::counter_offset();
2215 
2216   // Load up edx with the branch displacement
2217   if (is_wide) {
2218     __ movl(rdx, at_bcp(1));
2219   } else {
2220     __ load_signed_short(rdx, at_bcp(1));
2221   }
2222   __ bswapl(rdx);
2223 
2224   if (!is_wide) {
2225     __ sarl(rdx, 16);
2226   }
2227   LP64_ONLY(__ movl2ptr(rdx, rdx));
2228 
2229   // Handle all the JSR stuff here, then exit.
2230   // It's much shorter and cleaner than intermingling with the non-JSR
2231   // normal-branch stuff occurring below.
2232   if (is_jsr) {
2233     // Pre-load the next target bytecode into rbx
2234     __ load_unsigned_byte(rbx, Address(rbcp, rdx, Address::times_1, 0));
2235 
2236     // compute return address as bci in rax
2237     __ lea(rax, at_bcp((is_wide ? 5 : 3) -
2238                         in_bytes(ConstMethod::codes_offset())));
2239     __ subptr(rax, Address(rcx, Method::const_offset()));
2240     // Adjust the bcp in r13 by the displacement in rdx
2241     __ addptr(rbcp, rdx);
2242     // jsr returns atos that is not an oop
2243     __ push_i(rax);
2244     __ dispatch_only(vtos, true);
2245     return;
2246   }
2247 
2248   // Normal (non-jsr) branch handling
2249 
2250   // Adjust the bcp in r13 by the displacement in rdx
2251   __ addptr(rbcp, rdx);
2252 
2253   assert(UseLoopCounter || !UseOnStackReplacement,
2254          "on-stack-replacement requires loop counters");
2255   Label backedge_counter_overflow;
2256   Label dispatch;
2257   if (UseLoopCounter) {
2258     // increment backedge counter for backward branches
2259     // rax: MDO
2260     // rbx: MDO bumped taken-count
2261     // rcx: method
2262     // rdx: target offset
2263     // r13: target bcp
2264     // r14: locals pointer
2265     __ testl(rdx, rdx);             // check if forward or backward branch
2266     __ jcc(Assembler::positive, dispatch); // count only if backward branch
2267 
2268     // check if MethodCounters exists
2269     Label has_counters;
2270     __ movptr(rax, Address(rcx, Method::method_counters_offset()));
2271     __ testptr(rax, rax);
2272     __ jcc(Assembler::notZero, has_counters);
2273     __ push(rdx);
2274     __ push(rcx);
2275     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::build_method_counters),
2276                rcx);
2277     __ pop(rcx);
2278     __ pop(rdx);
2279     __ movptr(rax, Address(rcx, Method::method_counters_offset()));
2280     __ testptr(rax, rax);
2281     __ jcc(Assembler::zero, dispatch);
2282     __ bind(has_counters);
2283 
2284     Label no_mdo;
2285     if (ProfileInterpreter) {
2286       // Are we profiling?
2287       __ movptr(rbx, Address(rcx, in_bytes(Method::method_data_offset())));
2288       __ testptr(rbx, rbx);
2289       __ jccb(Assembler::zero, no_mdo);
2290       // Increment the MDO backedge counter
2291       const Address mdo_backedge_counter(rbx, in_bytes(MethodData::backedge_counter_offset()) +
2292           in_bytes(InvocationCounter::counter_offset()));
2293       const Address mask(rbx, in_bytes(MethodData::backedge_mask_offset()));
2294       __ increment_mask_and_jump(mdo_backedge_counter, mask, rax,
2295           UseOnStackReplacement ? &backedge_counter_overflow : nullptr);
2296       __ jmp(dispatch);
2297     }
2298     __ bind(no_mdo);
2299     // Increment backedge counter in MethodCounters*
2300     __ movptr(rcx, Address(rcx, Method::method_counters_offset()));
2301     const Address mask(rcx, in_bytes(MethodCounters::backedge_mask_offset()));
2302     __ increment_mask_and_jump(Address(rcx, be_offset), mask, rax,
2303         UseOnStackReplacement ? &backedge_counter_overflow : nullptr);
2304     __ bind(dispatch);
2305   }
2306 
2307   // Pre-load the next target bytecode into rbx
2308   __ load_unsigned_byte(rbx, Address(rbcp, 0));
2309 
2310   // continue with the bytecode @ target
2311   // rax: return bci for jsr's, unused otherwise
2312   // rbx: target bytecode
2313   // r13: target bcp
2314   __ dispatch_only(vtos, true);
2315 
2316   if (UseLoopCounter) {
2317     if (UseOnStackReplacement) {
2318       Label set_mdp;
2319       // invocation counter overflow
2320       __ bind(backedge_counter_overflow);
2321       __ negptr(rdx);
2322       __ addptr(rdx, rbcp); // branch bcp
2323       // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
2324       __ call_VM(noreg,
2325                  CAST_FROM_FN_PTR(address,
2326                                   InterpreterRuntime::frequency_counter_overflow),
2327                  rdx);
2328 
2329       // rax: osr nmethod (osr ok) or null (osr not possible)
2330       // rdx: scratch
2331       // r14: locals pointer
2332       // r13: bcp
2333       __ testptr(rax, rax);                        // test result
2334       __ jcc(Assembler::zero, dispatch);         // no osr if null
2335       // nmethod may have been invalidated (VM may block upon call_VM return)
2336       __ cmpb(Address(rax, nmethod::state_offset()), nmethod::in_use);
2337       __ jcc(Assembler::notEqual, dispatch);
2338 
2339       // We have the address of an on stack replacement routine in rax.
2340       // In preparation of invoking it, first we must migrate the locals
2341       // and monitors from off the interpreter frame on the stack.
2342       // Ensure to save the osr nmethod over the migration call,
2343       // it will be preserved in rbx.
2344       __ mov(rbx, rax);
2345 
2346       NOT_LP64(__ get_thread(rcx));
2347 
2348       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
2349 
2350       // rax is OSR buffer, move it to expected parameter location
2351       LP64_ONLY(__ mov(j_rarg0, rax));
2352       NOT_LP64(__ mov(rcx, rax));
2353       // We use j_rarg definitions here so that registers don't conflict as parameter
2354       // registers change across platforms as we are in the midst of a calling
2355       // sequence to the OSR nmethod and we don't want collision. These are NOT parameters.
2356 
2357       const Register retaddr   = LP64_ONLY(j_rarg2) NOT_LP64(rdi);
2358       const Register sender_sp = LP64_ONLY(j_rarg1) NOT_LP64(rdx);
2359 
2360       // pop the interpreter frame
2361       __ movptr(sender_sp, Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize)); // get sender sp
2362       __ leave();                                // remove frame anchor
2363       __ pop(retaddr);                           // get return address
2364       __ mov(rsp, sender_sp);                   // set sp to sender sp
2365       // Ensure compiled code always sees stack at proper alignment
2366       __ andptr(rsp, -(StackAlignmentInBytes));
2367 
2368       // unlike x86 we need no specialized return from compiled code
2369       // to the interpreter or the call stub.
2370 
2371       // push the return address
2372       __ push(retaddr);
2373 
2374       // and begin the OSR nmethod
2375       __ jmp(Address(rbx, nmethod::osr_entry_point_offset()));
2376     }
2377   }
2378 }
2379 
2380 void TemplateTable::if_0cmp(Condition cc) {
2381   transition(itos, vtos);
2382   // assume branch is more often taken than not (loops use backward branches)
2383   Label not_taken;
2384   __ testl(rax, rax);
2385   __ jcc(j_not(cc), not_taken);
2386   branch(false, false);
2387   __ bind(not_taken);
2388   __ profile_not_taken_branch(rax);
2389 }
2390 
2391 void TemplateTable::if_icmp(Condition cc) {
2392   transition(itos, vtos);
2393   // assume branch is more often taken than not (loops use backward branches)
2394   Label not_taken;
2395   __ pop_i(rdx);
2396   __ cmpl(rdx, rax);
2397   __ jcc(j_not(cc), not_taken);
2398   branch(false, false);
2399   __ bind(not_taken);
2400   __ profile_not_taken_branch(rax);
2401 }
2402 
2403 void TemplateTable::if_nullcmp(Condition cc) {
2404   transition(atos, vtos);
2405   // assume branch is more often taken than not (loops use backward branches)
2406   Label not_taken;
2407   __ testptr(rax, rax);
2408   __ jcc(j_not(cc), not_taken);
2409   branch(false, false);
2410   __ bind(not_taken);
2411   __ profile_not_taken_branch(rax);
2412 }
2413 
2414 void TemplateTable::if_acmp(Condition cc) {
2415   transition(atos, vtos);
2416   // assume branch is more often taken than not (loops use backward branches)
2417   Label taken, not_taken;
2418   __ pop_ptr(rdx);
2419 
2420   __ profile_acmp(rbx, rdx, rax, rcx);
2421 
2422   const int is_inline_type_mask = markWord::inline_type_pattern;
2423   if (EnableValhalla) {
2424     __ cmpoop(rdx, rax);
2425     __ jcc(Assembler::equal, (cc == equal) ? taken : not_taken);
2426 
2427     // might be substitutable, test if either rax or rdx is null
2428     __ testptr(rax, rax);
2429     __ jcc(Assembler::zero, (cc == equal) ? not_taken : taken);
2430     __ testptr(rdx, rdx);
2431     __ jcc(Assembler::zero, (cc == equal) ? not_taken : taken);
2432 
2433     // and both are values ?
2434     __ movptr(rbx, Address(rdx, oopDesc::mark_offset_in_bytes()));
2435     __ andptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
2436     __ andptr(rbx, is_inline_type_mask);
2437     __ cmpptr(rbx, is_inline_type_mask);
2438     __ jcc(Assembler::notEqual, (cc == equal) ? not_taken : taken);
2439 
2440     // same value klass ?
2441     __ load_metadata(rbx, rdx);
2442     __ load_metadata(rcx, rax);
2443     __ cmpptr(rbx, rcx);
2444     __ jcc(Assembler::notEqual, (cc == equal) ? not_taken : taken);
2445 
2446     // Know both are the same type, let's test for substitutability...
2447     if (cc == equal) {
2448       invoke_is_substitutable(rax, rdx, taken, not_taken);
2449     } else {
2450       invoke_is_substitutable(rax, rdx, not_taken, taken);
2451     }
2452     __ stop("Not reachable");
2453   }
2454 
2455   __ cmpoop(rdx, rax);
2456   __ jcc(j_not(cc), not_taken);
2457   __ bind(taken);
2458   branch(false, false);
2459   __ bind(not_taken);
2460   __ profile_not_taken_branch(rax, true);
2461 }
2462 
2463 void TemplateTable::invoke_is_substitutable(Register aobj, Register bobj,
2464                                             Label& is_subst, Label& not_subst) {
2465   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::is_substitutable), aobj, bobj);
2466   // Restored...rax answer, jmp to outcome...
2467   __ testl(rax, rax);
2468   __ jcc(Assembler::zero, not_subst);
2469   __ jmp(is_subst);
2470 }
2471 
2472 void TemplateTable::ret() {
2473   transition(vtos, vtos);
2474   locals_index(rbx);
2475   LP64_ONLY(__ movslq(rbx, iaddress(rbx))); // get return bci, compute return bcp
2476   NOT_LP64(__ movptr(rbx, iaddress(rbx)));
2477   __ profile_ret(rbx, rcx);
2478   __ get_method(rax);
2479   __ movptr(rbcp, Address(rax, Method::const_offset()));
2480   __ lea(rbcp, Address(rbcp, rbx, Address::times_1,
2481                       ConstMethod::codes_offset()));
2482   __ dispatch_next(vtos, 0, true);
2483 }
2484 
2485 void TemplateTable::wide_ret() {
2486   transition(vtos, vtos);
2487   locals_index_wide(rbx);
2488   __ movptr(rbx, aaddress(rbx)); // get return bci, compute return bcp
2489   __ profile_ret(rbx, rcx);
2490   __ get_method(rax);
2491   __ movptr(rbcp, Address(rax, Method::const_offset()));
2492   __ lea(rbcp, Address(rbcp, rbx, Address::times_1, ConstMethod::codes_offset()));
2493   __ dispatch_next(vtos, 0, true);
2494 }
2495 
2496 void TemplateTable::tableswitch() {
2497   Label default_case, continue_execution;
2498   transition(itos, vtos);
2499 
2500   // align r13/rsi
2501   __ lea(rbx, at_bcp(BytesPerInt));
2502   __ andptr(rbx, -BytesPerInt);
2503   // load lo & hi
2504   __ movl(rcx, Address(rbx, BytesPerInt));
2505   __ movl(rdx, Address(rbx, 2 * BytesPerInt));
2506   __ bswapl(rcx);
2507   __ bswapl(rdx);
2508   // check against lo & hi
2509   __ cmpl(rax, rcx);
2510   __ jcc(Assembler::less, default_case);
2511   __ cmpl(rax, rdx);
2512   __ jcc(Assembler::greater, default_case);
2513   // lookup dispatch offset
2514   __ subl(rax, rcx);
2515   __ movl(rdx, Address(rbx, rax, Address::times_4, 3 * BytesPerInt));
2516   __ profile_switch_case(rax, rbx, rcx);
2517   // continue execution
2518   __ bind(continue_execution);
2519   __ bswapl(rdx);
2520   LP64_ONLY(__ movl2ptr(rdx, rdx));
2521   __ load_unsigned_byte(rbx, Address(rbcp, rdx, Address::times_1));
2522   __ addptr(rbcp, rdx);
2523   __ dispatch_only(vtos, true);
2524   // handle default
2525   __ bind(default_case);
2526   __ profile_switch_default(rax);
2527   __ movl(rdx, Address(rbx, 0));
2528   __ jmp(continue_execution);
2529 }
2530 
2531 void TemplateTable::lookupswitch() {
2532   transition(itos, itos);
2533   __ stop("lookupswitch bytecode should have been rewritten");
2534 }
2535 
2536 void TemplateTable::fast_linearswitch() {
2537   transition(itos, vtos);
2538   Label loop_entry, loop, found, continue_execution;
2539   // bswap rax so we can avoid bswapping the table entries
2540   __ bswapl(rax);
2541   // align r13
2542   __ lea(rbx, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2543                                     // this instruction (change offsets
2544                                     // below)
2545   __ andptr(rbx, -BytesPerInt);
2546   // set counter
2547   __ movl(rcx, Address(rbx, BytesPerInt));
2548   __ bswapl(rcx);
2549   __ jmpb(loop_entry);
2550   // table search
2551   __ bind(loop);
2552   __ cmpl(rax, Address(rbx, rcx, Address::times_8, 2 * BytesPerInt));
2553   __ jcc(Assembler::equal, found);
2554   __ bind(loop_entry);
2555   __ decrementl(rcx);
2556   __ jcc(Assembler::greaterEqual, loop);
2557   // default case
2558   __ profile_switch_default(rax);
2559   __ movl(rdx, Address(rbx, 0));
2560   __ jmp(continue_execution);
2561   // entry found -> get offset
2562   __ bind(found);
2563   __ movl(rdx, Address(rbx, rcx, Address::times_8, 3 * BytesPerInt));
2564   __ profile_switch_case(rcx, rax, rbx);
2565   // continue execution
2566   __ bind(continue_execution);
2567   __ bswapl(rdx);
2568   __ movl2ptr(rdx, rdx);
2569   __ load_unsigned_byte(rbx, Address(rbcp, rdx, Address::times_1));
2570   __ addptr(rbcp, rdx);
2571   __ dispatch_only(vtos, true);
2572 }
2573 
2574 void TemplateTable::fast_binaryswitch() {
2575   transition(itos, vtos);
2576   // Implementation using the following core algorithm:
2577   //
2578   // int binary_search(int key, LookupswitchPair* array, int n) {
2579   //   // Binary search according to "Methodik des Programmierens" by
2580   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2581   //   int i = 0;
2582   //   int j = n;
2583   //   while (i+1 < j) {
2584   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2585   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2586   //     // where a stands for the array and assuming that the (inexisting)
2587   //     // element a[n] is infinitely big.
2588   //     int h = (i + j) >> 1;
2589   //     // i < h < j
2590   //     if (key < array[h].fast_match()) {
2591   //       j = h;
2592   //     } else {
2593   //       i = h;
2594   //     }
2595   //   }
2596   //   // R: a[i] <= key < a[i+1] or Q
2597   //   // (i.e., if key is within array, i is the correct index)
2598   //   return i;
2599   // }
2600 
2601   // Register allocation
2602   const Register key   = rax; // already set (tosca)
2603   const Register array = rbx;
2604   const Register i     = rcx;
2605   const Register j     = rdx;
2606   const Register h     = rdi;
2607   const Register temp  = rsi;
2608 
2609   // Find array start
2610   NOT_LP64(__ save_bcp());
2611 
2612   __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2613                                           // get rid of this
2614                                           // instruction (change
2615                                           // offsets below)
2616   __ andptr(array, -BytesPerInt);
2617 
2618   // Initialize i & j
2619   __ xorl(i, i);                            // i = 0;
2620   __ movl(j, Address(array, -BytesPerInt)); // j = length(array);
2621 
2622   // Convert j into native byteordering
2623   __ bswapl(j);
2624 
2625   // And start
2626   Label entry;
2627   __ jmp(entry);
2628 
2629   // binary search loop
2630   {
2631     Label loop;
2632     __ bind(loop);
2633     // int h = (i + j) >> 1;
2634     __ leal(h, Address(i, j, Address::times_1)); // h = i + j;
2635     __ sarl(h, 1);                               // h = (i + j) >> 1;
2636     // if (key < array[h].fast_match()) {
2637     //   j = h;
2638     // } else {
2639     //   i = h;
2640     // }
2641     // Convert array[h].match to native byte-ordering before compare
2642     __ movl(temp, Address(array, h, Address::times_8));
2643     __ bswapl(temp);
2644     __ cmpl(key, temp);
2645     // j = h if (key <  array[h].fast_match())
2646     __ cmov32(Assembler::less, j, h);
2647     // i = h if (key >= array[h].fast_match())
2648     __ cmov32(Assembler::greaterEqual, i, h);
2649     // while (i+1 < j)
2650     __ bind(entry);
2651     __ leal(h, Address(i, 1)); // i+1
2652     __ cmpl(h, j);             // i+1 < j
2653     __ jcc(Assembler::less, loop);
2654   }
2655 
2656   // end of binary search, result index is i (must check again!)
2657   Label default_case;
2658   // Convert array[i].match to native byte-ordering before compare
2659   __ movl(temp, Address(array, i, Address::times_8));
2660   __ bswapl(temp);
2661   __ cmpl(key, temp);
2662   __ jcc(Assembler::notEqual, default_case);
2663 
2664   // entry found -> j = offset
2665   __ movl(j , Address(array, i, Address::times_8, BytesPerInt));
2666   __ profile_switch_case(i, key, array);
2667   __ bswapl(j);
2668   LP64_ONLY(__ movslq(j, j));
2669 
2670   NOT_LP64(__ restore_bcp());
2671   NOT_LP64(__ restore_locals());                           // restore rdi
2672 
2673   __ load_unsigned_byte(rbx, Address(rbcp, j, Address::times_1));
2674   __ addptr(rbcp, j);
2675   __ dispatch_only(vtos, true);
2676 
2677   // default case -> j = default offset
2678   __ bind(default_case);
2679   __ profile_switch_default(i);
2680   __ movl(j, Address(array, -2 * BytesPerInt));
2681   __ bswapl(j);
2682   LP64_ONLY(__ movslq(j, j));
2683 
2684   NOT_LP64(__ restore_bcp());
2685   NOT_LP64(__ restore_locals());
2686 
2687   __ load_unsigned_byte(rbx, Address(rbcp, j, Address::times_1));
2688   __ addptr(rbcp, j);
2689   __ dispatch_only(vtos, true);
2690 }
2691 
2692 void TemplateTable::_return(TosState state) {
2693   transition(state, state);
2694 
2695   assert(_desc->calls_vm(),
2696          "inconsistent calls_vm information"); // call in remove_activation
2697 
2698   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2699     assert(state == vtos, "only valid state");
2700     Register robj = LP64_ONLY(c_rarg1) NOT_LP64(rax);
2701     __ movptr(robj, aaddress(0));
2702     __ load_klass(rdi, robj, rscratch1);
2703     __ testb(Address(rdi, Klass::misc_flags_offset()), KlassFlags::_misc_has_finalizer);
2704     Label skip_register_finalizer;
2705     __ jcc(Assembler::zero, skip_register_finalizer);
2706 
2707     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), robj);
2708 
2709     __ bind(skip_register_finalizer);
2710   }
2711 
2712   if (_desc->bytecode() != Bytecodes::_return_register_finalizer) {
2713     Label no_safepoint;
2714     NOT_PRODUCT(__ block_comment("Thread-local Safepoint poll"));
2715 #ifdef _LP64
2716     __ testb(Address(r15_thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
2717 #else
2718     const Register thread = rdi;
2719     __ get_thread(thread);
2720     __ testb(Address(thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
2721 #endif
2722     __ jcc(Assembler::zero, no_safepoint);
2723     __ push(state);
2724     __ push_cont_fastpath();
2725     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2726                                        InterpreterRuntime::at_safepoint));
2727     __ pop_cont_fastpath();
2728     __ pop(state);
2729     __ bind(no_safepoint);
2730   }
2731 
2732   // Narrow result if state is itos but result type is smaller.
2733   // Need to narrow in the return bytecode rather than in generate_return_entry
2734   // since compiled code callers expect the result to already be narrowed.
2735   if (state == itos) {
2736     __ narrow(rax);
2737   }
2738 
2739   __ remove_activation(state, rbcp, true, true, true);
2740 
2741   __ jmp(rbcp);
2742 }
2743 
2744 // ----------------------------------------------------------------------------
2745 // Volatile variables demand their effects be made known to all CPU's
2746 // in order.  Store buffers on most chips allow reads & writes to
2747 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2748 // without some kind of memory barrier (i.e., it's not sufficient that
2749 // the interpreter does not reorder volatile references, the hardware
2750 // also must not reorder them).
2751 //
2752 // According to the new Java Memory Model (JMM):
2753 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2754 //     writes act as acquire & release, so:
2755 // (2) A read cannot let unrelated NON-volatile memory refs that
2756 //     happen after the read float up to before the read.  It's OK for
2757 //     non-volatile memory refs that happen before the volatile read to
2758 //     float down below it.
2759 // (3) Similar a volatile write cannot let unrelated NON-volatile
2760 //     memory refs that happen BEFORE the write float down to after the
2761 //     write.  It's OK for non-volatile memory refs that happen after the
2762 //     volatile write to float up before it.
2763 //
2764 // We only put in barriers around volatile refs (they are expensive),
2765 // not _between_ memory refs (that would require us to track the
2766 // flavor of the previous memory refs).  Requirements (2) and (3)
2767 // require some barriers before volatile stores and after volatile
2768 // loads.  These nearly cover requirement (1) but miss the
2769 // volatile-store-volatile-load case.  This final case is placed after
2770 // volatile-stores although it could just as well go before
2771 // volatile-loads.
2772 
2773 void TemplateTable::volatile_barrier(Assembler::Membar_mask_bits order_constraint ) {
2774   // Helper function to insert a is-volatile test and memory barrier
2775   __ membar(order_constraint);
2776 }
2777 
2778 void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
2779                                                        Register cache,
2780                                                        Register index) {
2781   const Register temp = rbx;
2782   assert_different_registers(cache, index, temp);
2783 
2784   Label L_clinit_barrier_slow;
2785   Label resolved;
2786 
2787   Bytecodes::Code code = bytecode();
2788 
2789   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2790 
2791   __ load_method_entry(cache, index);
2792   switch(byte_no) {
2793     case f1_byte:
2794       __ load_unsigned_byte(temp, Address(cache, in_bytes(ResolvedMethodEntry::bytecode1_offset())));
2795       break;
2796     case f2_byte:
2797       __ load_unsigned_byte(temp, Address(cache, in_bytes(ResolvedMethodEntry::bytecode2_offset())));
2798       break;
2799     default:
2800       ShouldNotReachHere();
2801   }
2802   __ cmpl(temp, code);  // have we resolved this bytecode?
2803   __ jcc(Assembler::equal, resolved);
2804 
2805   // resolve first time through
2806   // Class initialization barrier slow path lands here as well.
2807   __ bind(L_clinit_barrier_slow);
2808   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2809   __ movl(temp, code);
2810   __ call_VM(noreg, entry, temp);
2811   // Update registers with resolved info
2812   __ load_method_entry(cache, index);
2813 
2814   __ bind(resolved);
2815 
2816   // Class initialization barrier for static methods
2817   if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
2818     const Register method = temp;
2819     const Register klass  = temp;
2820     const Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2821     assert(thread != noreg, "x86_32 not supported");
2822 
2823     __ movptr(method, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
2824     __ load_method_holder(klass, method);
2825     __ clinit_barrier(klass, thread, nullptr /*L_fast_path*/, &L_clinit_barrier_slow);
2826   }
2827 }
2828 
2829 void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
2830                                             Register cache,
2831                                             Register index) {
2832   const Register temp = rbx;
2833   assert_different_registers(cache, index, temp);
2834 
2835   Label resolved;
2836 
2837   Bytecodes::Code code = bytecode();
2838   switch (code) {
2839     case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2840     case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2841     default: break;
2842   }
2843 
2844   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2845   __ load_field_entry(cache, index);
2846   if (byte_no == f1_byte) {
2847     __ load_unsigned_byte(temp, Address(cache, in_bytes(ResolvedFieldEntry::get_code_offset())));
2848   } else {
2849     __ load_unsigned_byte(temp, Address(cache, in_bytes(ResolvedFieldEntry::put_code_offset())));
2850   }
2851   __ cmpl(temp, code);  // have we resolved this bytecode?
2852   __ jcc(Assembler::equal, resolved);
2853 
2854   // resolve first time through
2855   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2856   __ movl(temp, code);
2857   __ call_VM(noreg, entry, temp);
2858   // Update registers with resolved info
2859   __ load_field_entry(cache, index);
2860 
2861   __ bind(resolved);
2862 }
2863 
2864 void TemplateTable::load_resolved_field_entry(Register obj,
2865                                               Register cache,
2866                                               Register tos_state,
2867                                               Register offset,
2868                                               Register flags,
2869                                               bool is_static = false) {
2870   assert_different_registers(cache, tos_state, flags, offset);
2871 
2872   // Field offset
2873   __ load_sized_value(offset, Address(cache, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
2874 
2875   // Flags
2876   __ load_unsigned_byte(flags, Address(cache, in_bytes(ResolvedFieldEntry::flags_offset())));
2877 
2878   // TOS state
2879   __ load_unsigned_byte(tos_state, Address(cache, in_bytes(ResolvedFieldEntry::type_offset())));
2880 
2881   // Klass overwrite register
2882   if (is_static) {
2883     __ movptr(obj, Address(cache, ResolvedFieldEntry::field_holder_offset()));
2884     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2885     __ movptr(obj, Address(obj, mirror_offset));
2886     __ resolve_oop_handle(obj, rscratch2);
2887   }
2888 
2889 }
2890 
2891 void TemplateTable::load_invokedynamic_entry(Register method) {
2892   // setup registers
2893   const Register appendix = rax;
2894   const Register cache = rcx;
2895   const Register index = rdx;
2896   assert_different_registers(method, appendix, cache, index);
2897 
2898   __ save_bcp();
2899 
2900   Label resolved;
2901 
2902   __ load_resolved_indy_entry(cache, index);
2903   __ movptr(method, Address(cache, in_bytes(ResolvedIndyEntry::method_offset())));
2904 
2905   // Compare the method to zero
2906   __ testptr(method, method);
2907   __ jcc(Assembler::notZero, resolved);
2908 
2909   Bytecodes::Code code = bytecode();
2910 
2911   // Call to the interpreter runtime to resolve invokedynamic
2912   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2913   __ movl(method, code); // this is essentially Bytecodes::_invokedynamic
2914   __ call_VM(noreg, entry, method);
2915   // Update registers with resolved info
2916   __ load_resolved_indy_entry(cache, index);
2917   __ movptr(method, Address(cache, in_bytes(ResolvedIndyEntry::method_offset())));
2918 
2919 #ifdef ASSERT
2920   __ testptr(method, method);
2921   __ jcc(Assembler::notZero, resolved);
2922   __ stop("Should be resolved by now");
2923 #endif // ASSERT
2924   __ bind(resolved);
2925 
2926   Label L_no_push;
2927   // Check if there is an appendix
2928   __ load_unsigned_byte(index, Address(cache, in_bytes(ResolvedIndyEntry::flags_offset())));
2929   __ testl(index, (1 << ResolvedIndyEntry::has_appendix_shift));
2930   __ jcc(Assembler::zero, L_no_push);
2931 
2932   // Get appendix
2933   __ load_unsigned_short(index, Address(cache, in_bytes(ResolvedIndyEntry::resolved_references_index_offset())));
2934   // Push the appendix as a trailing parameter
2935   // since the parameter_size includes it.
2936   __ load_resolved_reference_at_index(appendix, index);
2937   __ verify_oop(appendix);
2938   __ push(appendix);  // push appendix (MethodType, CallSite, etc.)
2939   __ bind(L_no_push);
2940 
2941   // compute return type
2942   __ load_unsigned_byte(index, Address(cache, in_bytes(ResolvedIndyEntry::result_type_offset())));
2943   // load return address
2944   {
2945     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
2946     ExternalAddress table(table_addr);
2947 #ifdef _LP64
2948     __ lea(rscratch1, table);
2949     __ movptr(index, Address(rscratch1, index, Address::times_ptr));
2950 #else
2951     __ movptr(index, ArrayAddress(table, Address(noreg, index, Address::times_ptr)));
2952 #endif // _LP64
2953   }
2954 
2955   // push return address
2956   __ push(index);
2957 }
2958 
2959 void TemplateTable::load_resolved_method_entry_special_or_static(Register cache,
2960                                                                  Register method,
2961                                                                  Register flags) {
2962   // setup registers
2963   const Register index = rdx;
2964   assert_different_registers(cache, index);
2965   assert_different_registers(method, cache, flags);
2966 
2967   // determine constant pool cache field offsets
2968   resolve_cache_and_index_for_method(f1_byte, cache, index);
2969   __ load_unsigned_byte(flags, Address(cache, in_bytes(ResolvedMethodEntry::flags_offset())));
2970   __ movptr(method, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
2971 }
2972 
2973 void TemplateTable::load_resolved_method_entry_handle(Register cache,
2974                                                Register method,
2975                                                Register ref_index,
2976                                                Register flags) {
2977   // setup registers
2978   const Register index = rdx;
2979   assert_different_registers(cache, index);
2980   assert_different_registers(cache, method, ref_index, flags);
2981 
2982   // determine constant pool cache field offsets
2983   resolve_cache_and_index_for_method(f1_byte, cache, index);
2984   __ load_unsigned_byte(flags, Address(cache, in_bytes(ResolvedMethodEntry::flags_offset())));
2985 
2986   // Maybe push appendix
2987   Label L_no_push;
2988   __ testl(flags, (1 << ResolvedMethodEntry::has_appendix_shift));
2989   __ jcc(Assembler::zero, L_no_push);
2990   // invokehandle uses an index into the resolved references array
2991   __ load_unsigned_short(ref_index, Address(cache, in_bytes(ResolvedMethodEntry::resolved_references_index_offset())));
2992   // Push the appendix as a trailing parameter.
2993   // This must be done before we get the receiver,
2994   // since the parameter_size includes it.
2995   Register appendix = method;
2996   __ load_resolved_reference_at_index(appendix, ref_index);
2997   __ push(appendix);  // push appendix (MethodType, CallSite, etc.)
2998   __ bind(L_no_push);
2999 
3000   __ movptr(method, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
3001 }
3002 
3003 void TemplateTable::load_resolved_method_entry_interface(Register cache,
3004                                                          Register klass,
3005                                                          Register method_or_table_index,
3006                                                          Register flags) {
3007   // setup registers
3008   const Register index = rdx;
3009   assert_different_registers(cache, klass, method_or_table_index, flags);
3010 
3011   // determine constant pool cache field offsets
3012   resolve_cache_and_index_for_method(f1_byte, cache, index);
3013   __ load_unsigned_byte(flags, Address(cache, in_bytes(ResolvedMethodEntry::flags_offset())));
3014 
3015   // Invokeinterface can behave in different ways:
3016   // If calling a method from java.lang.Object, the forced virtual flag is true so the invocation will
3017   // behave like an invokevirtual call. The state of the virtual final flag will determine whether a method or
3018   // vtable index is placed in the register.
3019   // Otherwise, the registers will be populated with the klass and method.
3020 
3021   Label NotVirtual; Label NotVFinal; Label Done;
3022   __ testl(flags, 1 << ResolvedMethodEntry::is_forced_virtual_shift);
3023   __ jcc(Assembler::zero, NotVirtual);
3024   __ testl(flags, (1 << ResolvedMethodEntry::is_vfinal_shift));
3025   __ jcc(Assembler::zero, NotVFinal);
3026   __ movptr(method_or_table_index, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
3027   __ jmp(Done);
3028 
3029   __ bind(NotVFinal);
3030   __ load_unsigned_short(method_or_table_index, Address(cache, in_bytes(ResolvedMethodEntry::table_index_offset())));
3031   __ jmp(Done);
3032 
3033   __ bind(NotVirtual);
3034   __ movptr(method_or_table_index, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
3035   __ movptr(klass, Address(cache, in_bytes(ResolvedMethodEntry::klass_offset())));
3036   __ bind(Done);
3037 }
3038 
3039 void TemplateTable::load_resolved_method_entry_virtual(Register cache,
3040                                                        Register method_or_table_index,
3041                                                        Register flags) {
3042   // setup registers
3043   const Register index = rdx;
3044   assert_different_registers(index, cache);
3045   assert_different_registers(method_or_table_index, cache, flags);
3046 
3047   // determine constant pool cache field offsets
3048   resolve_cache_and_index_for_method(f2_byte, cache, index);
3049   __ load_unsigned_byte(flags, Address(cache, in_bytes(ResolvedMethodEntry::flags_offset())));
3050 
3051   // method_or_table_index can either be an itable index or a method depending on the virtual final flag
3052   Label isVFinal; Label Done;
3053   __ testl(flags, (1 << ResolvedMethodEntry::is_vfinal_shift));
3054   __ jcc(Assembler::notZero, isVFinal);
3055   __ load_unsigned_short(method_or_table_index, Address(cache, in_bytes(ResolvedMethodEntry::table_index_offset())));
3056   __ jmp(Done);
3057   __ bind(isVFinal);
3058   __ movptr(method_or_table_index, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
3059   __ bind(Done);
3060 }
3061 
3062 // The registers cache and index expected to be set before call.
3063 // Correct values of the cache and index registers are preserved.
3064 void TemplateTable::jvmti_post_field_access(Register cache,
3065                                             Register index,
3066                                             bool is_static,
3067                                             bool has_tos) {
3068   if (JvmtiExport::can_post_field_access()) {
3069     // Check to see if a field access watch has been set before we take
3070     // the time to call into the VM.
3071     Label L1;
3072     assert_different_registers(cache, index, rax);
3073     __ mov32(rax, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
3074     __ testl(rax,rax);
3075     __ jcc(Assembler::zero, L1);
3076 
3077     // cache entry pointer
3078     __ load_field_entry(cache, index);
3079     if (is_static) {
3080       __ xorptr(rax, rax);      // null object reference
3081     } else {
3082       __ pop(atos);         // Get the object
3083       __ verify_oop(rax);
3084       __ push(atos);        // Restore stack state
3085     }
3086     // rax,:   object pointer or null
3087     // cache: cache entry pointer
3088     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access),
3089               rax, cache);
3090 
3091     __ load_field_entry(cache, index);
3092     __ bind(L1);
3093   }
3094 }
3095 
3096 void TemplateTable::pop_and_check_object(Register r) {
3097   __ pop_ptr(r);
3098   __ null_check(r);  // for field access must check obj.
3099   __ verify_oop(r);
3100 }
3101 
3102 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
3103   transition(vtos, vtos);
3104 
3105   const Register obj   = LP64_ONLY(r9) NOT_LP64(rcx);
3106   const Register cache = rcx;
3107   const Register index = rdx;
3108   const Register off   = rbx;
3109   const Register tos_state   = rax;
3110   const Register flags = rdx;
3111   const Register bc    = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // uses same reg as obj, so don't mix them
3112 
3113   resolve_cache_and_index_for_field(byte_no, cache, index);
3114   jvmti_post_field_access(cache, index, is_static, false);
3115   load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
3116 
3117   const Address field(obj, off, Address::times_1, 0*wordSize);
3118 
3119   Label Done, notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj, notInlineType;
3120 
3121   // Make sure we don't need to mask edx after the above shift
3122   assert(btos == 0, "change code, btos != 0");
3123   __ testl(tos_state, tos_state);
3124   __ jcc(Assembler::notZero, notByte);
3125 
3126   // btos
3127   if (!is_static) pop_and_check_object(obj);
3128   __ access_load_at(T_BYTE, IN_HEAP, rax, field, noreg, noreg);
3129   __ push(btos);
3130   // Rewrite bytecode to be faster
3131   if (!is_static && rc == may_rewrite) {
3132     patch_bytecode(Bytecodes::_fast_bgetfield, bc, rbx);
3133   }
3134   __ jmp(Done);
3135 
3136   __ bind(notByte);
3137   __ cmpl(tos_state, ztos);
3138   __ jcc(Assembler::notEqual, notBool);
3139    if (!is_static) pop_and_check_object(obj);
3140   // ztos (same code as btos)
3141   __ access_load_at(T_BOOLEAN, IN_HEAP, rax, field, noreg, noreg);
3142   __ push(ztos);
3143   // Rewrite bytecode to be faster
3144   if (!is_static && rc == may_rewrite) {
3145     // use btos rewriting, no truncating to t/f bit is needed for getfield.
3146     patch_bytecode(Bytecodes::_fast_bgetfield, bc, rbx);
3147   }
3148   __ jmp(Done);
3149 
3150   __ bind(notBool);
3151   __ cmpl(tos_state, atos);
3152   __ jcc(Assembler::notEqual, notObj);
3153   // atos
3154   if (!EnableValhalla) {
3155     if (!is_static) pop_and_check_object(obj);
3156     do_oop_load(_masm, field, rax);
3157     __ push(atos);
3158     if (!is_static && rc == may_rewrite) {
3159       patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
3160     }
3161     __ jmp(Done);
3162   } else {
3163     if (is_static) {
3164       __ load_heap_oop(rax, field);
3165       Label is_null_free_inline_type, uninitialized;
3166       // Issue below if the static field has not been initialized yet
3167       __ test_field_is_null_free_inline_type(flags, rscratch1, is_null_free_inline_type);
3168         // field is not a null free inline type
3169         __ push(atos);
3170         __ jmp(Done);
3171       // field is a null free inline type, must not return null even if uninitialized
3172       __ bind(is_null_free_inline_type);
3173           __ testptr(rax, rax);
3174         __ jcc(Assembler::zero, uninitialized);
3175           __ push(atos);
3176           __ jmp(Done);
3177         __ bind(uninitialized);
3178 #ifdef _LP64
3179           Label slow_case, finish;
3180           __ movptr(rbx, Address(obj, java_lang_Class::klass_offset()));
3181           __ cmpb(Address(rbx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
3182           __ jcc(Assembler::notEqual, slow_case);
3183         __ get_default_value_oop(rbx, rscratch1, rax);
3184         __ jmp(finish);
3185         __ bind(slow_case);
3186 #endif // LP64
3187           __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_static_inline_type_field),
3188                 obj, cache);
3189 #ifdef _LP64
3190           __ bind(finish);
3191   #endif // _LP64
3192         __ verify_oop(rax);
3193         __ push(atos);
3194         __ jmp(Done);
3195     } else {
3196       Label is_flat, nonnull, is_inline_type, rewrite_inline, has_null_marker;
3197       __ test_field_is_null_free_inline_type(flags, rscratch1, is_inline_type);
3198       __ test_field_has_null_marker(flags, rscratch1, has_null_marker);
3199       // field is not a null free inline type
3200       pop_and_check_object(obj);
3201       __ load_heap_oop(rax, field);
3202       __ push(atos);
3203       if (rc == may_rewrite) {
3204         patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
3205       }
3206       __ jmp(Done);
3207       __ bind(is_inline_type);
3208       __ test_field_is_flat(flags, rscratch1, is_flat);
3209           // field is not flat
3210           pop_and_check_object(obj);
3211           __ load_heap_oop(rax, field);
3212           __ testptr(rax, rax);
3213           __ jcc(Assembler::notZero, nonnull);
3214             __ load_unsigned_short(flags, Address(cache, in_bytes(ResolvedFieldEntry::field_index_offset())));
3215             __ movptr(rcx, Address(cache, ResolvedFieldEntry::field_holder_offset()));
3216             __ get_inline_type_field_klass(rcx, flags, rbx);
3217             __ get_default_value_oop(rbx, rcx, rax);
3218           __ bind(nonnull);
3219           __ verify_oop(rax);
3220           __ push(atos);
3221           __ jmp(rewrite_inline);
3222         __ bind(is_flat);
3223           pop_and_check_object(rax);
3224           __ read_flat_field(rcx, rdx, rbx, rax);
3225           __ verify_oop(rax);
3226           __ push(atos);
3227           __ jmp(rewrite_inline);
3228       __ bind(has_null_marker);
3229         pop_and_check_object(rax);
3230         __ load_field_entry(rcx, rbx);
3231         call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_nullable_flat_field), rax, rcx);
3232         __ get_vm_result(rax, r15_thread);
3233         __ push(atos);
3234       __ bind(rewrite_inline);
3235       if (rc == may_rewrite) {
3236         patch_bytecode(Bytecodes::_fast_vgetfield, bc, rbx);
3237       }
3238         __ jmp(Done);
3239     }
3240   }
3241 
3242   __ bind(notObj);
3243 
3244   if (!is_static) pop_and_check_object(obj);
3245 
3246   __ cmpl(tos_state, itos);
3247   __ jcc(Assembler::notEqual, notInt);
3248   // itos
3249   __ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
3250   __ push(itos);
3251   // Rewrite bytecode to be faster
3252   if (!is_static && rc == may_rewrite) {
3253     patch_bytecode(Bytecodes::_fast_igetfield, bc, rbx);
3254   }
3255   __ jmp(Done);
3256 
3257   __ bind(notInt);
3258   __ cmpl(tos_state, ctos);
3259   __ jcc(Assembler::notEqual, notChar);
3260   // ctos
3261   __ access_load_at(T_CHAR, IN_HEAP, rax, field, noreg, noreg);
3262   __ push(ctos);
3263   // Rewrite bytecode to be faster
3264   if (!is_static && rc == may_rewrite) {
3265     patch_bytecode(Bytecodes::_fast_cgetfield, bc, rbx);
3266   }
3267   __ jmp(Done);
3268 
3269   __ bind(notChar);
3270   __ cmpl(tos_state, stos);
3271   __ jcc(Assembler::notEqual, notShort);
3272   // stos
3273   __ access_load_at(T_SHORT, IN_HEAP, rax, field, noreg, noreg);
3274   __ push(stos);
3275   // Rewrite bytecode to be faster
3276   if (!is_static && rc == may_rewrite) {
3277     patch_bytecode(Bytecodes::_fast_sgetfield, bc, rbx);
3278   }
3279   __ jmp(Done);
3280 
3281   __ bind(notShort);
3282   __ cmpl(tos_state, ltos);
3283   __ jcc(Assembler::notEqual, notLong);
3284   // ltos
3285     // Generate code as if volatile (x86_32).  There just aren't enough registers to
3286     // save that information and this code is faster than the test.
3287   __ access_load_at(T_LONG, IN_HEAP | MO_RELAXED, noreg /* ltos */, field, noreg, noreg);
3288   __ push(ltos);
3289   // Rewrite bytecode to be faster
3290   LP64_ONLY(if (!is_static && rc == may_rewrite) patch_bytecode(Bytecodes::_fast_lgetfield, bc, rbx));
3291   __ jmp(Done);
3292 
3293   __ bind(notLong);
3294   __ cmpl(tos_state, ftos);
3295   __ jcc(Assembler::notEqual, notFloat);
3296   // ftos
3297 
3298   __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
3299   __ push(ftos);
3300   // Rewrite bytecode to be faster
3301   if (!is_static && rc == may_rewrite) {
3302     patch_bytecode(Bytecodes::_fast_fgetfield, bc, rbx);
3303   }
3304   __ jmp(Done);
3305 
3306   __ bind(notFloat);
3307 #ifdef ASSERT
3308   Label notDouble;
3309   __ cmpl(tos_state, dtos);
3310   __ jcc(Assembler::notEqual, notDouble);
3311 #endif
3312   // dtos
3313   // MO_RELAXED: for the case of volatile field, in fact it adds no extra work for the underlying implementation
3314   __ access_load_at(T_DOUBLE, IN_HEAP | MO_RELAXED, noreg /* dtos */, field, noreg, noreg);
3315   __ push(dtos);
3316   // Rewrite bytecode to be faster
3317   if (!is_static && rc == may_rewrite) {
3318     patch_bytecode(Bytecodes::_fast_dgetfield, bc, rbx);
3319   }
3320 #ifdef ASSERT
3321   __ jmp(Done);
3322 
3323   __ bind(notDouble);
3324   __ stop("Bad state");
3325 #endif
3326 
3327   __ bind(Done);
3328   // [jk] not needed currently
3329   // volatile_barrier(Assembler::Membar_mask_bits(Assembler::LoadLoad |
3330   //                                              Assembler::LoadStore));
3331 }
3332 
3333 void TemplateTable::getfield(int byte_no) {
3334   getfield_or_static(byte_no, false);
3335 }
3336 
3337 void TemplateTable::nofast_getfield(int byte_no) {
3338   getfield_or_static(byte_no, false, may_not_rewrite);
3339 }
3340 
3341 void TemplateTable::getstatic(int byte_no) {
3342   getfield_or_static(byte_no, true);
3343 }
3344 
3345 // The registers cache and index expected to be set before call.
3346 // The function may destroy various registers, just not the cache and index registers.
3347 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
3348   // Cache is rcx and index is rdx
3349   const Register entry = LP64_ONLY(c_rarg2) NOT_LP64(rax); // ResolvedFieldEntry
3350   const Register obj = LP64_ONLY(c_rarg1) NOT_LP64(rbx);   // Object pointer
3351   const Register value = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // JValue object
3352 
3353   if (JvmtiExport::can_post_field_modification()) {
3354     // Check to see if a field modification watch has been set before
3355     // we take the time to call into the VM.
3356     Label L1;
3357     assert_different_registers(cache, obj, rax);
3358     __ mov32(rax, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
3359     __ testl(rax, rax);
3360     __ jcc(Assembler::zero, L1);
3361 
3362     __ mov(entry, cache);
3363 
3364     if (is_static) {
3365       // Life is simple.  Null out the object pointer.
3366       __ xorl(obj, obj);
3367 
3368     } else {
3369       // Life is harder. The stack holds the value on top, followed by
3370       // the object.  We don't know the size of the value, though; it
3371       // could be one or two words depending on its type. As a result,
3372       // we must find the type to determine where the object is.
3373 #ifndef _LP64
3374       Label two_word, valsize_known;
3375 #endif
3376       __ load_unsigned_byte(value, Address(entry, in_bytes(ResolvedFieldEntry::type_offset())));
3377 #ifdef _LP64
3378       __ movptr(obj, at_tos_p1());  // initially assume a one word jvalue
3379       __ cmpl(value, ltos);
3380       __ cmovptr(Assembler::equal,
3381                  obj, at_tos_p2()); // ltos (two word jvalue)
3382       __ cmpl(value, dtos);
3383       __ cmovptr(Assembler::equal,
3384                  obj, at_tos_p2()); // dtos (two word jvalue)
3385 #else
3386       __ mov(obj, rsp);
3387       __ cmpl(value, ltos);
3388       __ jccb(Assembler::equal, two_word);
3389       __ cmpl(value, dtos);
3390       __ jccb(Assembler::equal, two_word);
3391       __ addptr(obj, Interpreter::expr_offset_in_bytes(1)); // one word jvalue (not ltos, dtos)
3392       __ jmpb(valsize_known);
3393 
3394       __ bind(two_word);
3395       __ addptr(obj, Interpreter::expr_offset_in_bytes(2)); // two words jvalue
3396 
3397       __ bind(valsize_known);
3398       // setup object pointer
3399       __ movptr(obj, Address(obj, 0));
3400 #endif
3401     }
3402 
3403     // object (tos)
3404     __ mov(value, rsp);
3405     // obj: object pointer set up above (null if static)
3406     // cache: field entry pointer
3407     // value: jvalue object on the stack
3408     __ call_VM(noreg,
3409               CAST_FROM_FN_PTR(address,
3410                               InterpreterRuntime::post_field_modification),
3411               obj, entry, value);
3412     // Reload field entry
3413     __ load_field_entry(cache, index);
3414     __ bind(L1);
3415   }
3416 }
3417 
3418 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
3419   transition(vtos, vtos);
3420 
3421   const Register obj = rcx;
3422   const Register cache = rcx;
3423   const Register index = rdx;
3424   const Register tos_state   = rdx;
3425   const Register off   = rbx;
3426   const Register flags = r9;
3427 
3428   resolve_cache_and_index_for_field(byte_no, cache, index);
3429   jvmti_post_field_mod(cache, index, is_static);
3430   load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
3431 
3432   // [jk] not needed currently
3433   // volatile_barrier(Assembler::Membar_mask_bits(Assembler::LoadStore |
3434   //                                              Assembler::StoreStore));
3435 
3436   Label notVolatile, Done;
3437 
3438   // Check for volatile store
3439   __ movl(rscratch1, flags);
3440   __ andl(rscratch1, (1 << ResolvedFieldEntry::is_volatile_shift));
3441   __ testl(rscratch1, rscratch1);
3442   __ jcc(Assembler::zero, notVolatile);
3443 
3444   putfield_or_static_helper(byte_no, is_static, rc, obj, off, tos_state, flags);
3445   volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
3446                                                Assembler::StoreStore));
3447   __ jmp(Done);
3448   __ bind(notVolatile);
3449 
3450   putfield_or_static_helper(byte_no, is_static, rc, obj, off, tos_state, flags);
3451 
3452   __ bind(Done);
3453 }
3454 
3455 void TemplateTable::putfield_or_static_helper(int byte_no, bool is_static, RewriteControl rc,
3456                                               Register obj, Register off, Register tos_state, Register flags) {
3457 
3458   // field addresses
3459   const Address field(obj, off, Address::times_1, 0*wordSize);
3460   NOT_LP64( const Address hi(obj, off, Address::times_1, 1*wordSize);)
3461 
3462   Label notByte, notBool, notInt, notShort, notChar,
3463         notLong, notFloat, notObj, notInlineType;
3464   Label Done;
3465 
3466   const Register bc    = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
3467 
3468   // Test TOS state
3469   __ testl(tos_state, tos_state);
3470   __ jcc(Assembler::notZero, notByte);
3471 
3472   // btos
3473   {
3474     __ pop(btos);
3475     if (!is_static) pop_and_check_object(obj);
3476     __ access_store_at(T_BYTE, IN_HEAP, field, rax, noreg, noreg, noreg);
3477     if (!is_static && rc == may_rewrite) {
3478       patch_bytecode(Bytecodes::_fast_bputfield, bc, rbx, true, byte_no);
3479     }
3480     __ jmp(Done);
3481   }
3482 
3483   __ bind(notByte);
3484   __ cmpl(tos_state, ztos);
3485   __ jcc(Assembler::notEqual, notBool);
3486 
3487   // ztos
3488   {
3489     __ pop(ztos);
3490     if (!is_static) pop_and_check_object(obj);
3491     __ access_store_at(T_BOOLEAN, IN_HEAP, field, rax, noreg, noreg, noreg);
3492     if (!is_static && rc == may_rewrite) {
3493       patch_bytecode(Bytecodes::_fast_zputfield, bc, rbx, true, byte_no);
3494     }
3495     __ jmp(Done);
3496   }
3497 
3498   __ bind(notBool);
3499   __ cmpl(tos_state, atos);
3500   __ jcc(Assembler::notEqual, notObj);
3501 
3502   // atos
3503   {
3504     if (!EnableValhalla) {
3505       __ pop(atos);
3506       if (!is_static) pop_and_check_object(obj);
3507       // Store into the field
3508       do_oop_store(_masm, field, rax);
3509       if (!is_static && rc == may_rewrite) {
3510         patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
3511       }
3512       __ jmp(Done);
3513     } else {
3514       __ pop(atos);
3515       if (is_static) {
3516         Label is_inline_type;
3517         __ test_field_is_not_null_free_inline_type(flags, rscratch1, is_inline_type);
3518         __ null_check(rax);
3519         __ bind(is_inline_type);
3520         do_oop_store(_masm, field, rax);
3521         __ jmp(Done);
3522       } else {
3523         Label is_null_free_inline_type, is_flat, has_null_marker,
3524               write_null, rewrite_not_inline, rewrite_inline;
3525         __ test_field_is_null_free_inline_type(flags, rscratch1, is_null_free_inline_type);
3526         __ test_field_has_null_marker(flags, rscratch1, has_null_marker);
3527           // Not an inline type
3528           pop_and_check_object(obj);
3529           // Store into the field
3530           do_oop_store(_masm, field, rax);
3531           __ bind(rewrite_not_inline);
3532           if (rc == may_rewrite) {
3533             patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
3534           }
3535           __ jmp(Done);
3536         // Implementation of the inline type semantic
3537         __ bind(is_null_free_inline_type);
3538           __ null_check(rax);
3539           __ test_field_is_flat(flags, rscratch1, is_flat);
3540             // field is not flat
3541             pop_and_check_object(obj);
3542             // Store into the field
3543             do_oop_store(_masm, field, rax);
3544           __ jmp(rewrite_inline);
3545           __ bind(is_flat);
3546             // field is flat
3547             __ load_unsigned_short(rdx, Address(rcx, in_bytes(ResolvedFieldEntry::field_index_offset())));
3548             __ movptr(r9, Address(rcx, in_bytes(ResolvedFieldEntry::field_holder_offset())));
3549             pop_and_check_object(obj);  // obj = rcx
3550             __ load_klass(r8, rax, rscratch1);
3551             __ data_for_oop(rax, rax, r8);
3552             __ addptr(obj, off);
3553             __ inline_layout_info(r9, rdx, rbx);
3554             // because we use InlineLayoutInfo, we need special value access code specialized for fields (arrays will need a different API)
3555             __ flat_field_copy(IN_HEAP, rax, obj, rbx);
3556             __ jmp(rewrite_inline);
3557         __ bind(has_null_marker); // has null marker means the field is flat with a null marker
3558           pop_and_check_object(rbx);
3559           __ load_field_entry(rcx, rdx);
3560           call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_nullable_flat_field), rbx, rax, rcx);
3561         __ bind(rewrite_inline);
3562         if (rc == may_rewrite) {
3563           patch_bytecode(Bytecodes::_fast_vputfield, bc, rbx, true, byte_no);
3564         }
3565         __ jmp(Done);
3566       }
3567     }
3568   }
3569 
3570   __ bind(notObj);
3571   __ cmpl(tos_state, itos);
3572   __ jcc(Assembler::notEqual, notInt);
3573 
3574   // itos
3575   {
3576     __ pop(itos);
3577     if (!is_static) pop_and_check_object(obj);
3578     __ access_store_at(T_INT, IN_HEAP, field, rax, noreg, noreg, noreg);
3579     if (!is_static && rc == may_rewrite) {
3580       patch_bytecode(Bytecodes::_fast_iputfield, bc, rbx, true, byte_no);
3581     }
3582     __ jmp(Done);
3583   }
3584 
3585   __ bind(notInt);
3586   __ cmpl(tos_state, ctos);
3587   __ jcc(Assembler::notEqual, notChar);
3588 
3589   // ctos
3590   {
3591     __ pop(ctos);
3592     if (!is_static) pop_and_check_object(obj);
3593     __ access_store_at(T_CHAR, IN_HEAP, field, rax, noreg, noreg, noreg);
3594     if (!is_static && rc == may_rewrite) {
3595       patch_bytecode(Bytecodes::_fast_cputfield, bc, rbx, true, byte_no);
3596     }
3597     __ jmp(Done);
3598   }
3599 
3600   __ bind(notChar);
3601   __ cmpl(tos_state, stos);
3602   __ jcc(Assembler::notEqual, notShort);
3603 
3604   // stos
3605   {
3606     __ pop(stos);
3607     if (!is_static) pop_and_check_object(obj);
3608     __ access_store_at(T_SHORT, IN_HEAP, field, rax, noreg, noreg, noreg);
3609     if (!is_static && rc == may_rewrite) {
3610       patch_bytecode(Bytecodes::_fast_sputfield, bc, rbx, true, byte_no);
3611     }
3612     __ jmp(Done);
3613   }
3614 
3615   __ bind(notShort);
3616   __ cmpl(tos_state, ltos);
3617   __ jcc(Assembler::notEqual, notLong);
3618 
3619   // ltos
3620   {
3621     __ pop(ltos);
3622     if (!is_static) pop_and_check_object(obj);
3623     // MO_RELAXED: generate atomic store for the case of volatile field (important for x86_32)
3624     __ access_store_at(T_LONG, IN_HEAP | MO_RELAXED, field, noreg /* ltos*/, noreg, noreg, noreg);
3625 #ifdef _LP64
3626     if (!is_static && rc == may_rewrite) {
3627       patch_bytecode(Bytecodes::_fast_lputfield, bc, rbx, true, byte_no);
3628     }
3629 #endif // _LP64
3630     __ jmp(Done);
3631   }
3632 
3633   __ bind(notLong);
3634   __ cmpl(tos_state, ftos);
3635   __ jcc(Assembler::notEqual, notFloat);
3636 
3637   // ftos
3638   {
3639     __ pop(ftos);
3640     if (!is_static) pop_and_check_object(obj);
3641     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg, noreg);
3642     if (!is_static && rc == may_rewrite) {
3643       patch_bytecode(Bytecodes::_fast_fputfield, bc, rbx, true, byte_no);
3644     }
3645     __ jmp(Done);
3646   }
3647 
3648   __ bind(notFloat);
3649 #ifdef ASSERT
3650   Label notDouble;
3651   __ cmpl(tos_state, dtos);
3652   __ jcc(Assembler::notEqual, notDouble);
3653 #endif
3654 
3655   // dtos
3656   {
3657     __ pop(dtos);
3658     if (!is_static) pop_and_check_object(obj);
3659     // MO_RELAXED: for the case of volatile field, in fact it adds no extra work for the underlying implementation
3660     __ access_store_at(T_DOUBLE, IN_HEAP | MO_RELAXED, field, noreg /* dtos */, noreg, noreg, noreg);
3661     if (!is_static && rc == may_rewrite) {
3662       patch_bytecode(Bytecodes::_fast_dputfield, bc, rbx, true, byte_no);
3663     }
3664   }
3665 
3666 #ifdef ASSERT
3667   __ jmp(Done);
3668 
3669   __ bind(notDouble);
3670   __ stop("Bad state");
3671 #endif
3672 
3673   __ bind(Done);
3674 }
3675 
3676 void TemplateTable::putfield(int byte_no) {
3677   putfield_or_static(byte_no, false);
3678 }
3679 
3680 void TemplateTable::nofast_putfield(int byte_no) {
3681   putfield_or_static(byte_no, false, may_not_rewrite);
3682 }
3683 
3684 void TemplateTable::putstatic(int byte_no) {
3685   putfield_or_static(byte_no, true);
3686 }
3687 
3688 void TemplateTable::jvmti_post_fast_field_mod() {
3689 
3690   const Register scratch = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
3691 
3692   if (JvmtiExport::can_post_field_modification()) {
3693     // Check to see if a field modification watch has been set before
3694     // we take the time to call into the VM.
3695     Label L2;
3696     __ mov32(scratch, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
3697     __ testl(scratch, scratch);
3698     __ jcc(Assembler::zero, L2);
3699     __ pop_ptr(rbx);                  // copy the object pointer from tos
3700     __ verify_oop(rbx);
3701     __ push_ptr(rbx);                 // put the object pointer back on tos
3702     // Save tos values before call_VM() clobbers them. Since we have
3703     // to do it for every data type, we use the saved values as the
3704     // jvalue object.
3705     switch (bytecode()) {          // load values into the jvalue object
3706     case Bytecodes::_fast_vputfield: //fall through
3707     case Bytecodes::_fast_aputfield: __ push_ptr(rax); break;
3708     case Bytecodes::_fast_bputfield: // fall through
3709     case Bytecodes::_fast_zputfield: // fall through
3710     case Bytecodes::_fast_sputfield: // fall through
3711     case Bytecodes::_fast_cputfield: // fall through
3712     case Bytecodes::_fast_iputfield: __ push_i(rax); break;
3713     case Bytecodes::_fast_dputfield: __ push(dtos); break;
3714     case Bytecodes::_fast_fputfield: __ push(ftos); break;
3715     case Bytecodes::_fast_lputfield: __ push_l(rax); break;
3716 
3717     default:
3718       ShouldNotReachHere();
3719     }
3720     __ mov(scratch, rsp);             // points to jvalue on the stack
3721     // access constant pool cache entry
3722     LP64_ONLY(__ load_field_entry(c_rarg2, rax));
3723     NOT_LP64(__ load_field_entry(rax, rdx));
3724     __ verify_oop(rbx);
3725     // rbx: object pointer copied above
3726     // c_rarg2: cache entry pointer
3727     // c_rarg3: jvalue object on the stack
3728     LP64_ONLY(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), rbx, c_rarg2, c_rarg3));
3729     NOT_LP64(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), rbx, rax, rcx));
3730 
3731     switch (bytecode()) {             // restore tos values
3732     case Bytecodes::_fast_vputfield: // fall through
3733     case Bytecodes::_fast_aputfield: __ pop_ptr(rax); break;
3734     case Bytecodes::_fast_bputfield: // fall through
3735     case Bytecodes::_fast_zputfield: // fall through
3736     case Bytecodes::_fast_sputfield: // fall through
3737     case Bytecodes::_fast_cputfield: // fall through
3738     case Bytecodes::_fast_iputfield: __ pop_i(rax); break;
3739     case Bytecodes::_fast_dputfield: __ pop(dtos); break;
3740     case Bytecodes::_fast_fputfield: __ pop(ftos); break;
3741     case Bytecodes::_fast_lputfield: __ pop_l(rax); break;
3742     default: break;
3743     }
3744     __ bind(L2);
3745   }
3746 }
3747 
3748 void TemplateTable::fast_storefield(TosState state) {
3749   transition(state, vtos);
3750 
3751   Label notVolatile, Done;
3752 
3753   jvmti_post_fast_field_mod();
3754 
3755   __ push(rax);
3756   __ load_field_entry(rcx, rax);
3757   load_resolved_field_entry(noreg, rcx, rax, rbx, rdx);
3758   __ pop(rax);
3759   // RBX: field offset, RCX: RAX: TOS, RDX: flags
3760 
3761   // Get object from stack
3762   pop_and_check_object(rcx);
3763 
3764   // field address
3765   const Address field(rcx, rbx, Address::times_1);
3766 
3767   // Check for volatile store
3768   __ movl(rscratch2, rdx);  // saving flags for is_flat test
3769   __ andl(rscratch2, (1 << ResolvedFieldEntry::is_volatile_shift));
3770   __ testl(rscratch2, rscratch2);
3771   __ jcc(Assembler::zero, notVolatile);
3772 
3773   fast_storefield_helper(field, rax, rdx);
3774   volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
3775                                                Assembler::StoreStore));
3776   __ jmp(Done);
3777   __ bind(notVolatile);
3778 
3779   fast_storefield_helper(field, rax, rdx);
3780 
3781   __ bind(Done);
3782 }
3783 
3784 void TemplateTable::fast_storefield_helper(Address field, Register rax, Register flags) {
3785 
3786   // DANGER: 'field' argument depends on rcx and rbx
3787 
3788   // access field
3789   switch (bytecode()) {
3790   case Bytecodes::_fast_vputfield:
3791     {
3792       Label is_flat, has_null_marker, write_null, done;
3793       __ test_field_has_null_marker(flags, rscratch1, has_null_marker);
3794       // Null free field cases: flat or not flat
3795       __ null_check(rax);
3796       __ test_field_is_flat(flags, rscratch1, is_flat);
3797         // field is not flat
3798         do_oop_store(_masm, field, rax);
3799         __ jmp(done);
3800       __ bind(is_flat);
3801         __ load_field_entry(r8, r9);
3802         __ load_unsigned_short(r9, Address(r8, in_bytes(ResolvedFieldEntry::field_index_offset())));
3803         __ movptr(r8, Address(r8, in_bytes(ResolvedFieldEntry::field_holder_offset())));
3804         __ inline_layout_info(r8, r9, r8);
3805         __ load_klass(rdx, rax, rscratch1);
3806         __ data_for_oop(rax, rax, rdx);
3807         __ lea(rcx, field);
3808         __ flat_field_copy(IN_HEAP, rax, rcx, r8);
3809         __ jmp(done);
3810       __ bind(has_null_marker); // has null marker means the field is flat with a null marker
3811         __ movptr(rbx, rcx);
3812         __ load_field_entry(rcx, rdx);
3813         call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_nullable_flat_field), rbx, rax, rcx);
3814       __ bind(done);
3815     }
3816     break;
3817   case Bytecodes::_fast_aputfield:
3818     {
3819       do_oop_store(_masm, field, rax);
3820     }
3821     break;
3822   case Bytecodes::_fast_lputfield:
3823 #ifdef _LP64
3824     __ access_store_at(T_LONG, IN_HEAP, field, noreg /* ltos */, noreg, noreg, noreg);
3825 #else
3826   __ stop("should not be rewritten");
3827 #endif
3828     break;
3829   case Bytecodes::_fast_iputfield:
3830     __ access_store_at(T_INT, IN_HEAP, field, rax, noreg, noreg, noreg);
3831     break;
3832   case Bytecodes::_fast_zputfield:
3833     __ access_store_at(T_BOOLEAN, IN_HEAP, field, rax, noreg, noreg, noreg);
3834     break;
3835   case Bytecodes::_fast_bputfield:
3836     __ access_store_at(T_BYTE, IN_HEAP, field, rax, noreg, noreg, noreg);
3837     break;
3838   case Bytecodes::_fast_sputfield:
3839     __ access_store_at(T_SHORT, IN_HEAP, field, rax, noreg, noreg, noreg);
3840     break;
3841   case Bytecodes::_fast_cputfield:
3842     __ access_store_at(T_CHAR, IN_HEAP, field, rax, noreg, noreg, noreg);
3843     break;
3844   case Bytecodes::_fast_fputfield:
3845     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos*/, noreg, noreg, noreg);
3846     break;
3847   case Bytecodes::_fast_dputfield:
3848     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos*/, noreg, noreg, noreg);
3849     break;
3850   default:
3851     ShouldNotReachHere();
3852   }
3853 }
3854 
3855 void TemplateTable::fast_accessfield(TosState state) {
3856   transition(atos, state);
3857 
3858   // Do the JVMTI work here to avoid disturbing the register state below
3859   if (JvmtiExport::can_post_field_access()) {
3860     // Check to see if a field access watch has been set before we
3861     // take the time to call into the VM.
3862     Label L1;
3863     __ mov32(rcx, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
3864     __ testl(rcx, rcx);
3865     __ jcc(Assembler::zero, L1);
3866     // access constant pool cache entry
3867     LP64_ONLY(__ load_field_entry(c_rarg2, rcx));
3868     NOT_LP64(__ load_field_entry(rcx, rdx));
3869     __ verify_oop(rax);
3870     __ push_ptr(rax);  // save object pointer before call_VM() clobbers it
3871     LP64_ONLY(__ mov(c_rarg1, rax));
3872     // c_rarg1: object pointer copied above
3873     // c_rarg2: cache entry pointer
3874     LP64_ONLY(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access), c_rarg1, c_rarg2));
3875     NOT_LP64(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access), rax, rcx));
3876     __ pop_ptr(rax); // restore object pointer
3877     __ bind(L1);
3878   }
3879 
3880   // access constant pool cache
3881   __ load_field_entry(rcx, rbx);
3882   __ load_sized_value(rdx, Address(rcx, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
3883 
3884   // rax: object
3885   __ verify_oop(rax);
3886   __ null_check(rax);
3887   Address field(rax, rdx, Address::times_1);
3888 
3889   // access field
3890   switch (bytecode()) {
3891   case Bytecodes::_fast_vgetfield:
3892     {
3893       Label is_flat, nonnull, Done, has_null_marker;
3894       __ load_unsigned_byte(rscratch1, Address(rcx, in_bytes(ResolvedFieldEntry::flags_offset())));
3895       __ test_field_has_null_marker(rscratch1, rscratch2, has_null_marker);
3896       __ test_field_is_flat(rscratch1, rscratch2, is_flat);
3897         // field is not flat
3898         __ load_heap_oop(rax, field);
3899         __ testptr(rax, rax);
3900         __ jcc(Assembler::notZero, nonnull);
3901           __ load_unsigned_short(rdx, Address(rcx, in_bytes(ResolvedFieldEntry::field_index_offset())));
3902           __ movptr(rcx, Address(rcx, ResolvedFieldEntry::field_holder_offset()));
3903           __ get_inline_type_field_klass(rcx, rdx, rbx);
3904           __ get_default_value_oop(rbx, rcx, rax);
3905         __ bind(nonnull);
3906         __ verify_oop(rax);
3907         __ jmp(Done);
3908       __ bind(is_flat);
3909       // field is flat
3910         __ read_flat_field(rcx, rdx, rbx, rax);
3911         __ jmp(Done);
3912       __ bind(has_null_marker);
3913         // rax = instance, rcx = resolved entry
3914         call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_nullable_flat_field), rax, rcx);
3915         __ get_vm_result(rax, r15_thread);
3916       __ bind(Done);
3917       __ verify_oop(rax);
3918     }
3919     break;
3920   case Bytecodes::_fast_agetfield:
3921     do_oop_load(_masm, field, rax);
3922     __ verify_oop(rax);
3923     break;
3924   case Bytecodes::_fast_lgetfield:
3925 #ifdef _LP64
3926     __ access_load_at(T_LONG, IN_HEAP, noreg /* ltos */, field, noreg, noreg);
3927 #else
3928   __ stop("should not be rewritten");
3929 #endif
3930     break;
3931   case Bytecodes::_fast_igetfield:
3932     __ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
3933     break;
3934   case Bytecodes::_fast_bgetfield:
3935     __ access_load_at(T_BYTE, IN_HEAP, rax, field, noreg, noreg);
3936     break;
3937   case Bytecodes::_fast_sgetfield:
3938     __ access_load_at(T_SHORT, IN_HEAP, rax, field, noreg, noreg);
3939     break;
3940   case Bytecodes::_fast_cgetfield:
3941     __ access_load_at(T_CHAR, IN_HEAP, rax, field, noreg, noreg);
3942     break;
3943   case Bytecodes::_fast_fgetfield:
3944     __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
3945     break;
3946   case Bytecodes::_fast_dgetfield:
3947     __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* dtos */, field, noreg, noreg);
3948     break;
3949   default:
3950     ShouldNotReachHere();
3951   }
3952   // [jk] not needed currently
3953   //   Label notVolatile;
3954   //   __ testl(rdx, rdx);
3955   //   __ jcc(Assembler::zero, notVolatile);
3956   //   __ membar(Assembler::LoadLoad);
3957   //   __ bind(notVolatile);
3958 }
3959 
3960 void TemplateTable::fast_xaccess(TosState state) {
3961   transition(vtos, state);
3962 
3963   // get receiver
3964   __ movptr(rax, aaddress(0));
3965   // access constant pool cache
3966   __ load_field_entry(rcx, rdx, 2);
3967   __ load_sized_value(rbx, Address(rcx, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
3968 
3969   // make sure exception is reported in correct bcp range (getfield is
3970   // next instruction)
3971   __ increment(rbcp);
3972   __ null_check(rax);
3973   const Address field = Address(rax, rbx, Address::times_1, 0*wordSize);
3974   switch (state) {
3975   case itos:
3976     __ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
3977     break;
3978   case atos:
3979     do_oop_load(_masm, field, rax);
3980     __ verify_oop(rax);
3981     break;
3982   case ftos:
3983     __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
3984     break;
3985   default:
3986     ShouldNotReachHere();
3987   }
3988 
3989   // [jk] not needed currently
3990   // Label notVolatile;
3991   // __ movl(rdx, Address(rcx, rdx, Address::times_8,
3992   //                      in_bytes(ConstantPoolCache::base_offset() +
3993   //                               ConstantPoolCacheEntry::flags_offset())));
3994   // __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift);
3995   // __ testl(rdx, 0x1);
3996   // __ jcc(Assembler::zero, notVolatile);
3997   // __ membar(Assembler::LoadLoad);
3998   // __ bind(notVolatile);
3999 
4000   __ decrement(rbcp);
4001 }
4002 
4003 //-----------------------------------------------------------------------------
4004 // Calls
4005 
4006 void TemplateTable::prepare_invoke(Register cache, Register recv, Register flags) {
4007   // determine flags
4008   const Bytecodes::Code code = bytecode();
4009   const bool load_receiver       = (code != Bytecodes::_invokestatic) && (code != Bytecodes::_invokedynamic);
4010   assert_different_registers(recv, flags);
4011 
4012   // save 'interpreter return address'
4013   __ save_bcp();
4014 
4015   // Save flags and load TOS
4016   __ movl(rbcp, flags);
4017   __ load_unsigned_byte(flags, Address(cache, in_bytes(ResolvedMethodEntry::type_offset())));
4018 
4019   // load receiver if needed (after appendix is pushed so parameter size is correct)
4020   // Note: no return address pushed yet
4021   if (load_receiver) {
4022     __ load_unsigned_short(recv, Address(cache, in_bytes(ResolvedMethodEntry::num_parameters_offset())));
4023     const int no_return_pc_pushed_yet = -1;  // argument slot correction before we push return address
4024     const int receiver_is_at_end      = -1;  // back off one slot to get receiver
4025     Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
4026     __ movptr(recv, recv_addr);
4027     __ verify_oop(recv);
4028   }
4029 
4030   // load return address
4031   {
4032     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
4033     ExternalAddress table(table_addr);
4034 #ifdef _LP64
4035     __ lea(rscratch1, table);
4036     __ movptr(flags, Address(rscratch1, flags, Address::times_ptr));
4037 #else
4038     __ movptr(flags, ArrayAddress(table, Address(noreg, flags, Address::times_ptr)));
4039 #endif // _LP64
4040   }
4041 
4042   // push return address
4043   __ push(flags);
4044 
4045   // Restore flags value from the constant pool cache entry, and restore rsi
4046   // for later null checks.  r13 is the bytecode pointer
4047   __ movl(flags, rbcp);
4048   __ restore_bcp();
4049 }
4050 
4051 void TemplateTable::invokevirtual_helper(Register index,
4052                                          Register recv,
4053                                          Register flags) {
4054   // Uses temporary registers rax, rdx
4055   assert_different_registers(index, recv, rax, rdx);
4056   assert(index == rbx, "");
4057   assert(recv  == rcx, "");
4058 
4059   // Test for an invoke of a final method
4060   Label notFinal;
4061   __ movl(rax, flags);
4062   __ andl(rax, (1 << ResolvedMethodEntry::is_vfinal_shift));
4063   __ jcc(Assembler::zero, notFinal);
4064 
4065   const Register method = index;  // method must be rbx
4066   assert(method == rbx,
4067          "Method* must be rbx for interpreter calling convention");
4068 
4069   // do the call - the index is actually the method to call
4070   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
4071 
4072   // It's final, need a null check here!
4073   __ null_check(recv);
4074 
4075   // profile this call
4076   __ profile_final_call(rax);
4077   __ profile_arguments_type(rax, method, rbcp, true);
4078 
4079   __ jump_from_interpreted(method, rax);
4080 
4081   __ bind(notFinal);
4082 
4083   // get receiver klass
4084   __ load_klass(rax, recv, rscratch1);
4085 
4086   // profile this call
4087   __ profile_virtual_call(rax, rlocals, rdx);
4088   // get target Method* & entry point
4089   __ lookup_virtual_method(rax, index, method);
4090 
4091   __ profile_arguments_type(rdx, method, rbcp, true);
4092   __ jump_from_interpreted(method, rdx);
4093 }
4094 
4095 void TemplateTable::invokevirtual(int byte_no) {
4096   transition(vtos, vtos);
4097   assert(byte_no == f2_byte, "use this argument");
4098 
4099   load_resolved_method_entry_virtual(rcx,  // ResolvedMethodEntry*
4100                                      rbx,  // Method or itable index
4101                                      rdx); // Flags
4102   prepare_invoke(rcx,  // ResolvedMethodEntry*
4103                  rcx,  // Receiver
4104                  rdx); // flags
4105 
4106   // rbx: index
4107   // rcx: receiver
4108   // rdx: flags
4109   invokevirtual_helper(rbx, rcx, rdx);
4110 }
4111 
4112 void TemplateTable::invokespecial(int byte_no) {
4113   transition(vtos, vtos);
4114   assert(byte_no == f1_byte, "use this argument");
4115 
4116   load_resolved_method_entry_special_or_static(rcx,  // ResolvedMethodEntry*
4117                                                rbx,  // Method*
4118                                                rdx); // flags
4119   prepare_invoke(rcx,
4120                  rcx,  // get receiver also for null check
4121                  rdx); // flags
4122 
4123   __ verify_oop(rcx);
4124   __ null_check(rcx);
4125   // do the call
4126   __ profile_call(rax);
4127   __ profile_arguments_type(rax, rbx, rbcp, false);
4128   __ jump_from_interpreted(rbx, rax);
4129 }
4130 
4131 void TemplateTable::invokestatic(int byte_no) {
4132   transition(vtos, vtos);
4133   assert(byte_no == f1_byte, "use this argument");
4134 
4135   load_resolved_method_entry_special_or_static(rcx, // ResolvedMethodEntry*
4136                                                rbx, // Method*
4137                                                rdx  // flags
4138                                                );
4139   prepare_invoke(rcx, rcx, rdx);  // cache and flags
4140 
4141   // do the call
4142   __ profile_call(rax);
4143   __ profile_arguments_type(rax, rbx, rbcp, false);
4144   __ jump_from_interpreted(rbx, rax);
4145 }
4146 
4147 
4148 void TemplateTable::fast_invokevfinal(int byte_no) {
4149   transition(vtos, vtos);
4150   assert(byte_no == f2_byte, "use this argument");
4151   __ stop("fast_invokevfinal not used on x86");
4152 }
4153 
4154 
4155 void TemplateTable::invokeinterface(int byte_no) {
4156   transition(vtos, vtos);
4157   assert(byte_no == f1_byte, "use this argument");
4158 
4159   load_resolved_method_entry_interface(rcx,  // ResolvedMethodEntry*
4160                                        rax,  // Klass*
4161                                        rbx,  // Method* or itable/vtable index
4162                                        rdx); // flags
4163   prepare_invoke(rcx, rcx, rdx); // receiver, flags
4164 
4165   // First check for Object case, then private interface method,
4166   // then regular interface method.
4167 
4168   // Special case of invokeinterface called for virtual method of
4169   // java.lang.Object.  See cpCache.cpp for details.
4170   Label notObjectMethod;
4171   __ movl(rlocals, rdx);
4172   __ andl(rlocals, (1 << ResolvedMethodEntry::is_forced_virtual_shift));
4173   __ jcc(Assembler::zero, notObjectMethod);
4174 
4175   invokevirtual_helper(rbx, rcx, rdx);
4176   // no return from above
4177   __ bind(notObjectMethod);
4178 
4179   Label no_such_interface; // for receiver subtype check
4180   Register recvKlass; // used for exception processing
4181 
4182   // Check for private method invocation - indicated by vfinal
4183   Label notVFinal;
4184   __ movl(rlocals, rdx);
4185   __ andl(rlocals, (1 << ResolvedMethodEntry::is_vfinal_shift));
4186   __ jcc(Assembler::zero, notVFinal);
4187 
4188   // Get receiver klass into rlocals - also a null check
4189   __ load_klass(rlocals, rcx, rscratch1);
4190 
4191   Label subtype;
4192   __ check_klass_subtype(rlocals, rax, rbcp, subtype);
4193   // If we get here the typecheck failed
4194   recvKlass = rdx;
4195   __ mov(recvKlass, rlocals); // shuffle receiver class for exception use
4196   __ jmp(no_such_interface);
4197 
4198   __ bind(subtype);
4199 
4200   // do the call - rbx is actually the method to call
4201 
4202   __ profile_final_call(rdx);
4203   __ profile_arguments_type(rdx, rbx, rbcp, true);
4204 
4205   __ jump_from_interpreted(rbx, rdx);
4206   // no return from above
4207   __ bind(notVFinal);
4208 
4209   // Get receiver klass into rdx - also a null check
4210   __ restore_locals();  // restore r14
4211   __ load_klass(rdx, rcx, rscratch1);
4212 
4213   Label no_such_method;
4214 
4215   // Preserve method for throw_AbstractMethodErrorVerbose.
4216   __ mov(rcx, rbx);
4217   // Receiver subtype check against REFC.
4218   // Superklass in rax. Subklass in rdx. Blows rcx, rdi.
4219   __ lookup_interface_method(// inputs: rec. class, interface, itable index
4220                              rdx, rax, noreg,
4221                              // outputs: scan temp. reg, scan temp. reg
4222                              rbcp, rlocals,
4223                              no_such_interface,
4224                              /*return_method=*/false);
4225 
4226   // profile this call
4227   __ restore_bcp(); // rbcp was destroyed by receiver type check
4228   __ profile_virtual_call(rdx, rbcp, rlocals);
4229 
4230   // Get declaring interface class from method, and itable index
4231   __ load_method_holder(rax, rbx);
4232   __ movl(rbx, Address(rbx, Method::itable_index_offset()));
4233   __ subl(rbx, Method::itable_index_max);
4234   __ negl(rbx);
4235 
4236   // Preserve recvKlass for throw_AbstractMethodErrorVerbose.
4237   __ mov(rlocals, rdx);
4238   __ lookup_interface_method(// inputs: rec. class, interface, itable index
4239                              rlocals, rax, rbx,
4240                              // outputs: method, scan temp. reg
4241                              rbx, rbcp,
4242                              no_such_interface);
4243 
4244   // rbx: Method* to call
4245   // rcx: receiver
4246   // Check for abstract method error
4247   // Note: This should be done more efficiently via a throw_abstract_method_error
4248   //       interpreter entry point and a conditional jump to it in case of a null
4249   //       method.
4250   __ testptr(rbx, rbx);
4251   __ jcc(Assembler::zero, no_such_method);
4252 
4253   __ profile_arguments_type(rdx, rbx, rbcp, true);
4254 
4255   // do the call
4256   // rcx: receiver
4257   // rbx,: Method*
4258   __ jump_from_interpreted(rbx, rdx);
4259   __ should_not_reach_here();
4260 
4261   // exception handling code follows...
4262   // note: must restore interpreter registers to canonical
4263   //       state for exception handling to work correctly!
4264 
4265   __ bind(no_such_method);
4266   // throw exception
4267   __ pop(rbx);           // pop return address (pushed by prepare_invoke)
4268   __ restore_bcp();      // rbcp must be correct for exception handler   (was destroyed)
4269   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
4270   // Pass arguments for generating a verbose error message.
4271 #ifdef _LP64
4272   recvKlass = c_rarg1;
4273   Register method    = c_rarg2;
4274   if (recvKlass != rdx) { __ movq(recvKlass, rdx); }
4275   if (method != rcx)    { __ movq(method, rcx);    }
4276 #else
4277   recvKlass = rdx;
4278   Register method    = rcx;
4279 #endif
4280   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodErrorVerbose),
4281              recvKlass, method);
4282   // The call_VM checks for exception, so we should never return here.
4283   __ should_not_reach_here();
4284 
4285   __ bind(no_such_interface);
4286   // throw exception
4287   __ pop(rbx);           // pop return address (pushed by prepare_invoke)
4288   __ restore_bcp();      // rbcp must be correct for exception handler   (was destroyed)
4289   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
4290   // Pass arguments for generating a verbose error message.
4291   LP64_ONLY( if (recvKlass != rdx) { __ movq(recvKlass, rdx); } )
4292   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_IncompatibleClassChangeErrorVerbose),
4293              recvKlass, rax);
4294   // the call_VM checks for exception, so we should never return here.
4295   __ should_not_reach_here();
4296 }
4297 
4298 void TemplateTable::invokehandle(int byte_no) {
4299   transition(vtos, vtos);
4300   assert(byte_no == f1_byte, "use this argument");
4301   const Register rbx_method = rbx;
4302   const Register rax_mtype  = rax;
4303   const Register rcx_recv   = rcx;
4304   const Register rdx_flags  = rdx;
4305 
4306   load_resolved_method_entry_handle(rcx, rbx_method, rax_mtype, rdx_flags);
4307   prepare_invoke(rcx, rcx_recv, rdx_flags);
4308 
4309   __ verify_method_ptr(rbx_method);
4310   __ verify_oop(rcx_recv);
4311   __ null_check(rcx_recv);
4312 
4313   // rax: MethodType object (from cpool->resolved_references[f1], if necessary)
4314   // rbx: MH.invokeExact_MT method
4315 
4316   // Note:  rax_mtype is already pushed (if necessary)
4317 
4318   // FIXME: profile the LambdaForm also
4319   __ profile_final_call(rax);
4320   __ profile_arguments_type(rdx, rbx_method, rbcp, true);
4321 
4322   __ jump_from_interpreted(rbx_method, rdx);
4323 }
4324 
4325 void TemplateTable::invokedynamic(int byte_no) {
4326   transition(vtos, vtos);
4327   assert(byte_no == f1_byte, "use this argument");
4328 
4329   const Register rbx_method   = rbx;
4330   const Register rax_callsite = rax;
4331 
4332   load_invokedynamic_entry(rbx_method);
4333   // rax: CallSite object (from cpool->resolved_references[])
4334   // rbx: MH.linkToCallSite method
4335 
4336   // Note:  rax_callsite is already pushed
4337 
4338   // %%% should make a type profile for any invokedynamic that takes a ref argument
4339   // profile this call
4340   __ profile_call(rbcp);
4341   __ profile_arguments_type(rdx, rbx_method, rbcp, false);
4342 
4343   __ verify_oop(rax_callsite);
4344 
4345   __ jump_from_interpreted(rbx_method, rdx);
4346 }
4347 
4348 //-----------------------------------------------------------------------------
4349 // Allocation
4350 
4351 void TemplateTable::_new() {
4352   transition(vtos, atos);
4353   __ get_unsigned_2_byte_index_at_bcp(rdx, 1);
4354   Label slow_case;
4355   Label done;
4356 
4357   __ get_cpool_and_tags(rcx, rax);
4358 
4359   // Make sure the class we're about to instantiate has been resolved.
4360   // This is done before loading InstanceKlass to be consistent with the order
4361   // how Constant Pool is updated (see ConstantPool::klass_at_put)
4362   const int tags_offset = Array<u1>::base_offset_in_bytes();
4363   __ cmpb(Address(rax, rdx, Address::times_1, tags_offset), JVM_CONSTANT_Class);
4364   __ jcc(Assembler::notEqual, slow_case);
4365 
4366   // get InstanceKlass
4367   __ load_resolved_klass_at_index(rcx, rcx, rdx);
4368 
4369   // make sure klass is initialized
4370 #ifdef _LP64
4371   assert(VM_Version::supports_fast_class_init_checks(), "must support fast class initialization checks");
4372   __ clinit_barrier(rcx, r15_thread, nullptr /*L_fast_path*/, &slow_case);
4373 #else
4374   __ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
4375   __ jcc(Assembler::notEqual, slow_case);
4376 #endif
4377 
4378   __ allocate_instance(rcx, rax, rdx, rbx, true, slow_case);
4379     if (DTraceAllocProbes) {
4380       // Trigger dtrace event for fastpath
4381       __ push(atos);
4382       __ call_VM_leaf(
4383            CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), rax);
4384       __ pop(atos);
4385     }
4386   __ jmp(done);
4387 
4388   // slow case
4389   __ bind(slow_case);
4390 
4391   Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rax);
4392   Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
4393 
4394   __ get_constant_pool(rarg1);
4395   __ get_unsigned_2_byte_index_at_bcp(rarg2, 1);
4396   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), rarg1, rarg2);
4397    __ verify_oop(rax);
4398 
4399   // continue
4400   __ bind(done);
4401 }
4402 
4403 void TemplateTable::newarray() {
4404   transition(itos, atos);
4405   Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rdx);
4406   __ load_unsigned_byte(rarg1, at_bcp(1));
4407   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
4408           rarg1, rax);
4409 }
4410 
4411 void TemplateTable::anewarray() {
4412   transition(itos, atos);
4413 
4414   Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rcx);
4415   Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
4416 
4417   __ get_unsigned_2_byte_index_at_bcp(rarg2, 1);
4418   __ get_constant_pool(rarg1);
4419   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
4420           rarg1, rarg2, rax);
4421 }
4422 
4423 void TemplateTable::arraylength() {
4424   transition(atos, itos);
4425   __ movl(rax, Address(rax, arrayOopDesc::length_offset_in_bytes()));
4426 }
4427 
4428 void TemplateTable::checkcast() {
4429   transition(atos, atos);
4430   Label done, is_null, ok_is_subtype, quicked, resolved;
4431   __ testptr(rax, rax); // object is in rax
4432   __ jcc(Assembler::zero, is_null);
4433 
4434   // Get cpool & tags index
4435   __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
4436   __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
4437   // See if bytecode has already been quicked
4438   __ movzbl(rdx, Address(rdx, rbx,
4439       Address::times_1,
4440       Array<u1>::base_offset_in_bytes()));
4441   __ cmpl(rdx, JVM_CONSTANT_Class);
4442   __ jcc(Assembler::equal, quicked);
4443   __ push(atos); // save receiver for result, and for GC
4444   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4445 
4446   // vm_result_2 has metadata result
4447 #ifndef _LP64
4448   // borrow rdi from locals
4449   __ get_thread(rdi);
4450   __ get_vm_result_2(rax, rdi);
4451   __ restore_locals();
4452 #else
4453   __ get_vm_result_2(rax, r15_thread);
4454 #endif
4455 
4456   __ pop_ptr(rdx); // restore receiver
4457   __ jmpb(resolved);
4458 
4459   // Get superklass in rax and subklass in rbx
4460   __ bind(quicked);
4461   __ mov(rdx, rax); // Save object in rdx; rax needed for subtype check
4462   __ load_resolved_klass_at_index(rax, rcx, rbx);
4463 
4464   __ bind(resolved);
4465   __ load_klass(rbx, rdx, rscratch1);
4466 
4467   // Generate subtype check.  Blows rcx, rdi.  Object in rdx.
4468   // Superklass in rax.  Subklass in rbx.
4469   __ gen_subtype_check(rbx, ok_is_subtype);
4470 
4471   // Come here on failure
4472   __ push_ptr(rdx);
4473   // object is at TOS
4474   __ jump(RuntimeAddress(Interpreter::_throw_ClassCastException_entry));
4475 
4476   // Come here on success
4477   __ bind(ok_is_subtype);
4478   __ mov(rax, rdx); // Restore object in rdx
4479   __ jmp(done);
4480 
4481   __ bind(is_null);
4482 
4483   // Collect counts on whether this check-cast sees nulls a lot or not.
4484   if (ProfileInterpreter) {
4485     __ profile_null_seen(rcx);
4486   }
4487 
4488   __ bind(done);
4489 }
4490 
4491 void TemplateTable::instanceof() {
4492   transition(atos, itos);
4493   Label done, is_null, ok_is_subtype, quicked, resolved;
4494   __ testptr(rax, rax);
4495   __ jcc(Assembler::zero, is_null);
4496 
4497   // Get cpool & tags index
4498   __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
4499   __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
4500   // See if bytecode has already been quicked
4501   __ movzbl(rdx, Address(rdx, rbx,
4502         Address::times_1,
4503         Array<u1>::base_offset_in_bytes()));
4504   __ cmpl(rdx, JVM_CONSTANT_Class);
4505   __ jcc(Assembler::equal, quicked);
4506 
4507   __ push(atos); // save receiver for result, and for GC
4508   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4509   // vm_result_2 has metadata result
4510 
4511 #ifndef _LP64
4512   // borrow rdi from locals
4513   __ get_thread(rdi);
4514   __ get_vm_result_2(rax, rdi);
4515   __ restore_locals();
4516 #else
4517   __ get_vm_result_2(rax, r15_thread);
4518 #endif
4519 
4520   __ pop_ptr(rdx); // restore receiver
4521   __ verify_oop(rdx);
4522   __ load_klass(rdx, rdx, rscratch1);
4523   __ jmpb(resolved);
4524 
4525   // Get superklass in rax and subklass in rdx
4526   __ bind(quicked);
4527   __ load_klass(rdx, rax, rscratch1);
4528   __ load_resolved_klass_at_index(rax, rcx, rbx);
4529 
4530   __ bind(resolved);
4531 
4532   // Generate subtype check.  Blows rcx, rdi
4533   // Superklass in rax.  Subklass in rdx.
4534   __ gen_subtype_check(rdx, ok_is_subtype);
4535 
4536   // Come here on failure
4537   __ xorl(rax, rax);
4538   __ jmpb(done);
4539   // Come here on success
4540   __ bind(ok_is_subtype);
4541   __ movl(rax, 1);
4542 
4543   // Collect counts on whether this test sees nulls a lot or not.
4544   if (ProfileInterpreter) {
4545     __ jmp(done);
4546     __ bind(is_null);
4547     __ profile_null_seen(rcx);
4548   } else {
4549     __ bind(is_null);   // same as 'done'
4550   }
4551   __ bind(done);
4552   // rax = 0: obj == nullptr or  obj is not an instanceof the specified klass
4553   // rax = 1: obj != nullptr and obj is     an instanceof the specified klass
4554 }
4555 
4556 //----------------------------------------------------------------------------------------------------
4557 // Breakpoints
4558 void TemplateTable::_breakpoint() {
4559   // Note: We get here even if we are single stepping..
4560   // jbug insists on setting breakpoints at every bytecode
4561   // even if we are in single step mode.
4562 
4563   transition(vtos, vtos);
4564 
4565   Register rarg = LP64_ONLY(c_rarg1) NOT_LP64(rcx);
4566 
4567   // get the unpatched byte code
4568   __ get_method(rarg);
4569   __ call_VM(noreg,
4570              CAST_FROM_FN_PTR(address,
4571                               InterpreterRuntime::get_original_bytecode_at),
4572              rarg, rbcp);
4573   __ mov(rbx, rax);  // why?
4574 
4575   // post the breakpoint event
4576   __ get_method(rarg);
4577   __ call_VM(noreg,
4578              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
4579              rarg, rbcp);
4580 
4581   // complete the execution of original bytecode
4582   __ dispatch_only_normal(vtos);
4583 }
4584 
4585 //-----------------------------------------------------------------------------
4586 // Exceptions
4587 
4588 void TemplateTable::athrow() {
4589   transition(atos, vtos);
4590   __ null_check(rax);
4591   __ jump(RuntimeAddress(Interpreter::throw_exception_entry()));
4592 }
4593 
4594 //-----------------------------------------------------------------------------
4595 // Synchronization
4596 //
4597 // Note: monitorenter & exit are symmetric routines; which is reflected
4598 //       in the assembly code structure as well
4599 //
4600 // Stack layout:
4601 //
4602 // [expressions  ] <--- rsp               = expression stack top
4603 // ..
4604 // [expressions  ]
4605 // [monitor entry] <--- monitor block top = expression stack bot
4606 // ..
4607 // [monitor entry]
4608 // [frame data   ] <--- monitor block bot
4609 // ...
4610 // [saved rbp    ] <--- rbp
4611 void TemplateTable::monitorenter() {
4612   transition(atos, vtos);
4613 
4614   // check for null object
4615   __ null_check(rax);
4616 
4617   Label is_inline_type;
4618   __ movptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
4619   __ test_markword_is_inline_type(rbx, is_inline_type);
4620 
4621   const Address monitor_block_top(
4622         rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4623   const Address monitor_block_bot(
4624         rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
4625   const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
4626 
4627   Label allocated;
4628 
4629   Register rtop = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
4630   Register rbot = LP64_ONLY(c_rarg2) NOT_LP64(rbx);
4631   Register rmon = LP64_ONLY(c_rarg1) NOT_LP64(rdx);
4632 
4633   // initialize entry pointer
4634   __ xorl(rmon, rmon); // points to free slot or null
4635 
4636   // find a free slot in the monitor block (result in rmon)
4637   {
4638     Label entry, loop, exit;
4639     __ movptr(rtop, monitor_block_top); // derelativize pointer
4640     __ lea(rtop, Address(rbp, rtop, Address::times_ptr));
4641     // rtop points to current entry, starting with top-most entry
4642 
4643     __ lea(rbot, monitor_block_bot);    // points to word before bottom
4644                                         // of monitor block
4645     __ jmpb(entry);
4646 
4647     __ bind(loop);
4648     // check if current entry is used
4649     __ cmpptr(Address(rtop, BasicObjectLock::obj_offset()), NULL_WORD);
4650     // if not used then remember entry in rmon
4651     __ cmovptr(Assembler::equal, rmon, rtop);   // cmov => cmovptr
4652     // check if current entry is for same object
4653     __ cmpptr(rax, Address(rtop, BasicObjectLock::obj_offset()));
4654     // if same object then stop searching
4655     __ jccb(Assembler::equal, exit);
4656     // otherwise advance to next entry
4657     __ addptr(rtop, entry_size);
4658     __ bind(entry);
4659     // check if bottom reached
4660     __ cmpptr(rtop, rbot);
4661     // if not at bottom then check this entry
4662     __ jcc(Assembler::notEqual, loop);
4663     __ bind(exit);
4664   }
4665 
4666   __ testptr(rmon, rmon); // check if a slot has been found
4667   __ jcc(Assembler::notZero, allocated); // if found, continue with that one
4668 
4669   // allocate one if there's no free slot
4670   {
4671     Label entry, loop;
4672     // 1. compute new pointers          // rsp: old expression stack top
4673     __ movptr(rmon, monitor_block_bot); // rmon: old expression stack bottom
4674     __ lea(rmon, Address(rbp, rmon, Address::times_ptr));
4675     __ subptr(rsp, entry_size);         // move expression stack top
4676     __ subptr(rmon, entry_size);        // move expression stack bottom
4677     __ mov(rtop, rsp);                  // set start value for copy loop
4678     __ subptr(monitor_block_bot, entry_size / wordSize); // set new monitor block bottom
4679     __ jmp(entry);
4680     // 2. move expression stack contents
4681     __ bind(loop);
4682     __ movptr(rbot, Address(rtop, entry_size)); // load expression stack
4683                                                 // word from old location
4684     __ movptr(Address(rtop, 0), rbot);          // and store it at new location
4685     __ addptr(rtop, wordSize);                  // advance to next word
4686     __ bind(entry);
4687     __ cmpptr(rtop, rmon);                      // check if bottom reached
4688     __ jcc(Assembler::notEqual, loop);          // if not at bottom then
4689                                                 // copy next word
4690   }
4691 
4692   // call run-time routine
4693   // rmon: points to monitor entry
4694   __ bind(allocated);
4695 
4696   // Increment bcp to point to the next bytecode, so exception
4697   // handling for async. exceptions work correctly.
4698   // The object has already been popped from the stack, so the
4699   // expression stack looks correct.
4700   __ increment(rbcp);
4701 
4702   // store object
4703   __ movptr(Address(rmon, BasicObjectLock::obj_offset()), rax);
4704   __ lock_object(rmon);
4705 
4706   // check to make sure this monitor doesn't cause stack overflow after locking
4707   __ save_bcp();  // in case of exception
4708   __ generate_stack_overflow_check(0);
4709 
4710   // The bcp has already been incremented. Just need to dispatch to
4711   // next instruction.
4712   __ dispatch_next(vtos);
4713 
4714   __ bind(is_inline_type);
4715   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4716                     InterpreterRuntime::throw_identity_exception), rax);
4717   __ should_not_reach_here();
4718 }
4719 
4720 void TemplateTable::monitorexit() {
4721   transition(atos, vtos);
4722 
4723   // check for null object
4724   __ null_check(rax);
4725 
4726   const int is_inline_type_mask = markWord::inline_type_pattern;
4727   Label has_identity;
4728   __ movptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
4729   __ andptr(rbx, is_inline_type_mask);
4730   __ cmpl(rbx, is_inline_type_mask);
4731   __ jcc(Assembler::notEqual, has_identity);
4732   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4733                      InterpreterRuntime::throw_illegal_monitor_state_exception));
4734   __ should_not_reach_here();
4735   __ bind(has_identity);
4736 
4737   const Address monitor_block_top(
4738         rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4739   const Address monitor_block_bot(
4740         rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
4741   const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
4742 
4743   Register rtop = LP64_ONLY(c_rarg1) NOT_LP64(rdx);
4744   Register rbot = LP64_ONLY(c_rarg2) NOT_LP64(rbx);
4745 
4746   Label found;
4747 
4748   // find matching slot
4749   {
4750     Label entry, loop;
4751     __ movptr(rtop, monitor_block_top); // derelativize pointer
4752     __ lea(rtop, Address(rbp, rtop, Address::times_ptr));
4753     // rtop points to current entry, starting with top-most entry
4754 
4755     __ lea(rbot, monitor_block_bot);    // points to word before bottom
4756                                         // of monitor block
4757     __ jmpb(entry);
4758 
4759     __ bind(loop);
4760     // check if current entry is for same object
4761     __ cmpptr(rax, Address(rtop, BasicObjectLock::obj_offset()));
4762     // if same object then stop searching
4763     __ jcc(Assembler::equal, found);
4764     // otherwise advance to next entry
4765     __ addptr(rtop, entry_size);
4766     __ bind(entry);
4767     // check if bottom reached
4768     __ cmpptr(rtop, rbot);
4769     // if not at bottom then check this entry
4770     __ jcc(Assembler::notEqual, loop);
4771   }
4772 
4773   // error handling. Unlocking was not block-structured
4774   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4775                    InterpreterRuntime::throw_illegal_monitor_state_exception));
4776   __ should_not_reach_here();
4777 
4778   // call run-time routine
4779   __ bind(found);
4780   __ push_ptr(rax); // make sure object is on stack (contract with oopMaps)
4781   __ unlock_object(rtop);
4782   __ pop_ptr(rax); // discard object
4783 }
4784 
4785 // Wide instructions
4786 void TemplateTable::wide() {
4787   transition(vtos, vtos);
4788   __ load_unsigned_byte(rbx, at_bcp(1));
4789   ExternalAddress wtable((address)Interpreter::_wentry_point);
4790   __ jump(ArrayAddress(wtable, Address(noreg, rbx, Address::times_ptr)), rscratch1);
4791   // Note: the rbcp increment step is part of the individual wide bytecode implementations
4792 }
4793 
4794 // Multi arrays
4795 void TemplateTable::multianewarray() {
4796   transition(vtos, atos);
4797 
4798   Register rarg = LP64_ONLY(c_rarg1) NOT_LP64(rax);
4799   __ load_unsigned_byte(rax, at_bcp(3)); // get number of dimensions
4800   // last dim is on top of stack; we want address of first one:
4801   // first_addr = last_addr + (ndims - 1) * stackElementSize - 1*wordsize
4802   // the latter wordSize to point to the beginning of the array.
4803   __ lea(rarg, Address(rsp, rax, Interpreter::stackElementScale(), -wordSize));
4804   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray), rarg);
4805   __ load_unsigned_byte(rbx, at_bcp(3));
4806   __ lea(rsp, Address(rsp, rbx, Interpreter::stackElementScale()));  // get rid of counts
4807 }