1 /*
   2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "compiler/disassembler.hpp"
  28 #include "gc/shared/collectedHeap.hpp"
  29 #include "gc/shared/gc_globals.hpp"
  30 #include "gc/shared/tlab_globals.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "interpreter/interpreterRuntime.hpp"
  33 #include "interpreter/interp_masm.hpp"
  34 #include "interpreter/templateTable.hpp"
  35 #include "memory/universe.hpp"
  36 #include "oops/methodData.hpp"
  37 #include "oops/objArrayKlass.hpp"
  38 #include "oops/oop.inline.hpp"
  39 #include "prims/jvmtiExport.hpp"
  40 #include "prims/methodHandles.hpp"
  41 #include "runtime/frame.inline.hpp"
  42 #include "runtime/safepointMechanism.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "runtime/stubRoutines.hpp"
  45 #include "runtime/synchronizer.hpp"
  46 #include "utilities/macros.hpp"
  47 
  48 #define __ Disassembler::hook<InterpreterMacroAssembler>(__FILE__, __LINE__, _masm)->
  49 
  50 // Global Register Names
  51 static const Register rbcp     = LP64_ONLY(r13) NOT_LP64(rsi);
  52 static const Register rlocals  = LP64_ONLY(r14) NOT_LP64(rdi);
  53 
  54 // Address Computation: local variables
  55 static inline Address iaddress(int n) {
  56   return Address(rlocals, Interpreter::local_offset_in_bytes(n));
  57 }
  58 
  59 static inline Address laddress(int n) {
  60   return iaddress(n + 1);
  61 }
  62 
  63 #ifndef _LP64
  64 static inline Address haddress(int n) {
  65   return iaddress(n + 0);
  66 }
  67 #endif
  68 
  69 static inline Address faddress(int n) {
  70   return iaddress(n);
  71 }
  72 
  73 static inline Address daddress(int n) {
  74   return laddress(n);
  75 }
  76 
  77 static inline Address aaddress(int n) {
  78   return iaddress(n);
  79 }
  80 
  81 static inline Address iaddress(Register r) {
  82   return Address(rlocals, r, Address::times_ptr);
  83 }
  84 
  85 static inline Address laddress(Register r) {
  86   return Address(rlocals, r, Address::times_ptr, Interpreter::local_offset_in_bytes(1));
  87 }
  88 
  89 #ifndef _LP64
  90 static inline Address haddress(Register r)       {
  91   return Address(rlocals, r, Interpreter::stackElementScale(), Interpreter::local_offset_in_bytes(0));
  92 }
  93 #endif
  94 
  95 static inline Address faddress(Register r) {
  96   return iaddress(r);
  97 }
  98 
  99 static inline Address daddress(Register r) {
 100   return laddress(r);
 101 }
 102 
 103 static inline Address aaddress(Register r) {
 104   return iaddress(r);
 105 }
 106 
 107 
 108 // expression stack
 109 // (Note: Must not use symmetric equivalents at_rsp_m1/2 since they store
 110 // data beyond the rsp which is potentially unsafe in an MT environment;
 111 // an interrupt may overwrite that data.)
 112 static inline Address at_rsp   () {
 113   return Address(rsp, 0);
 114 }
 115 
 116 // At top of Java expression stack which may be different than esp().  It
 117 // isn't for category 1 objects.
 118 static inline Address at_tos   () {
 119   return Address(rsp,  Interpreter::expr_offset_in_bytes(0));
 120 }
 121 
 122 static inline Address at_tos_p1() {
 123   return Address(rsp,  Interpreter::expr_offset_in_bytes(1));
 124 }
 125 
 126 static inline Address at_tos_p2() {
 127   return Address(rsp,  Interpreter::expr_offset_in_bytes(2));
 128 }
 129 
 130 // Condition conversion
 131 static Assembler::Condition j_not(TemplateTable::Condition cc) {
 132   switch (cc) {
 133   case TemplateTable::equal        : return Assembler::notEqual;
 134   case TemplateTable::not_equal    : return Assembler::equal;
 135   case TemplateTable::less         : return Assembler::greaterEqual;
 136   case TemplateTable::less_equal   : return Assembler::greater;
 137   case TemplateTable::greater      : return Assembler::lessEqual;
 138   case TemplateTable::greater_equal: return Assembler::less;
 139   }
 140   ShouldNotReachHere();
 141   return Assembler::zero;
 142 }
 143 
 144 
 145 
 146 // Miscellaneous helper routines
 147 // Store an oop (or null) at the address described by obj.
 148 // If val == noreg this means store a null
 149 
 150 
 151 static void do_oop_store(InterpreterMacroAssembler* _masm,
 152                          Address dst,
 153                          Register val,
 154                          DecoratorSet decorators = 0) {
 155   assert(val == noreg || val == rax, "parameter is just for looks");
 156   __ store_heap_oop(dst, val,
 157                     NOT_LP64(rdx) LP64_ONLY(rscratch2),
 158                     NOT_LP64(rbx) LP64_ONLY(r9),
 159                     NOT_LP64(rsi) LP64_ONLY(r8), decorators);
 160 }
 161 
 162 static void do_oop_load(InterpreterMacroAssembler* _masm,
 163                         Address src,
 164                         Register dst,
 165                         DecoratorSet decorators = 0) {
 166   __ load_heap_oop(dst, src, rdx, rbx, decorators);
 167 }
 168 
 169 Address TemplateTable::at_bcp(int offset) {
 170   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 171   return Address(rbcp, offset);
 172 }
 173 
 174 
 175 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 176                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 177                                    int byte_no) {
 178   if (!RewriteBytecodes)  return;
 179   Label L_patch_done;
 180 
 181   switch (bc) {
 182   case Bytecodes::_fast_aputfield:
 183   case Bytecodes::_fast_bputfield:
 184   case Bytecodes::_fast_zputfield:
 185   case Bytecodes::_fast_cputfield:
 186   case Bytecodes::_fast_dputfield:
 187   case Bytecodes::_fast_fputfield:
 188   case Bytecodes::_fast_iputfield:
 189   case Bytecodes::_fast_lputfield:
 190   case Bytecodes::_fast_sputfield:
 191     {
 192       // We skip bytecode quickening for putfield instructions when
 193       // the put_code written to the constant pool cache is zero.
 194       // This is required so that every execution of this instruction
 195       // calls out to InterpreterRuntime::resolve_get_put to do
 196       // additional, required work.
 197       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 198       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 199       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 200       __ movl(bc_reg, bc);
 201       __ cmpl(temp_reg, (int) 0);
 202       __ jcc(Assembler::zero, L_patch_done);  // don't patch
 203     }
 204     break;
 205   default:
 206     assert(byte_no == -1, "sanity");
 207     // the pair bytecodes have already done the load.
 208     if (load_bc_into_bc_reg) {
 209       __ movl(bc_reg, bc);
 210     }
 211   }
 212 
 213   if (JvmtiExport::can_post_breakpoint()) {
 214     Label L_fast_patch;
 215     // if a breakpoint is present we can't rewrite the stream directly
 216     __ movzbl(temp_reg, at_bcp(0));
 217     __ cmpl(temp_reg, Bytecodes::_breakpoint);
 218     __ jcc(Assembler::notEqual, L_fast_patch);
 219     __ get_method(temp_reg);
 220     // Let breakpoint table handling rewrite to quicker bytecode
 221     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), temp_reg, rbcp, bc_reg);
 222 #ifndef ASSERT
 223     __ jmpb(L_patch_done);
 224 #else
 225     __ jmp(L_patch_done);
 226 #endif
 227     __ bind(L_fast_patch);
 228   }
 229 
 230 #ifdef ASSERT
 231   Label L_okay;
 232   __ load_unsigned_byte(temp_reg, at_bcp(0));
 233   __ cmpl(temp_reg, (int) Bytecodes::java_code(bc));
 234   __ jcc(Assembler::equal, L_okay);
 235   __ cmpl(temp_reg, bc_reg);
 236   __ jcc(Assembler::equal, L_okay);
 237   __ stop("patching the wrong bytecode");
 238   __ bind(L_okay);
 239 #endif
 240 
 241   // patch bytecode
 242   __ movb(at_bcp(0), bc_reg);
 243   __ bind(L_patch_done);
 244 }
 245 // Individual instructions
 246 
 247 
 248 void TemplateTable::nop() {
 249   transition(vtos, vtos);
 250   // nothing to do
 251 }
 252 
 253 void TemplateTable::shouldnotreachhere() {
 254   transition(vtos, vtos);
 255   __ stop("shouldnotreachhere bytecode");
 256 }
 257 
 258 void TemplateTable::aconst_null() {
 259   transition(vtos, atos);
 260   __ xorl(rax, rax);
 261 }
 262 
 263 void TemplateTable::iconst(int value) {
 264   transition(vtos, itos);
 265   if (value == 0) {
 266     __ xorl(rax, rax);
 267   } else {
 268     __ movl(rax, value);
 269   }
 270 }
 271 
 272 void TemplateTable::lconst(int value) {
 273   transition(vtos, ltos);
 274   if (value == 0) {
 275     __ xorl(rax, rax);
 276   } else {
 277     __ movl(rax, value);
 278   }
 279 #ifndef _LP64
 280   assert(value >= 0, "check this code");
 281   __ xorptr(rdx, rdx);
 282 #endif
 283 }
 284 
 285 
 286 
 287 void TemplateTable::fconst(int value) {
 288   transition(vtos, ftos);
 289   if (UseSSE >= 1) {
 290     static float one = 1.0f, two = 2.0f;
 291     switch (value) {
 292     case 0:
 293       __ xorps(xmm0, xmm0);
 294       break;
 295     case 1:
 296       __ movflt(xmm0, ExternalAddress((address) &one), rscratch1);
 297       break;
 298     case 2:
 299       __ movflt(xmm0, ExternalAddress((address) &two), rscratch1);
 300       break;
 301     default:
 302       ShouldNotReachHere();
 303       break;
 304     }
 305   } else {
 306 #ifdef _LP64
 307     ShouldNotReachHere();
 308 #else
 309            if (value == 0) { __ fldz();
 310     } else if (value == 1) { __ fld1();
 311     } else if (value == 2) { __ fld1(); __ fld1(); __ faddp(); // should do a better solution here
 312     } else                 { ShouldNotReachHere();
 313     }
 314 #endif // _LP64
 315   }
 316 }
 317 
 318 void TemplateTable::dconst(int value) {
 319   transition(vtos, dtos);
 320   if (UseSSE >= 2) {
 321     static double one = 1.0;
 322     switch (value) {
 323     case 0:
 324       __ xorpd(xmm0, xmm0);
 325       break;
 326     case 1:
 327       __ movdbl(xmm0, ExternalAddress((address) &one), rscratch1);
 328       break;
 329     default:
 330       ShouldNotReachHere();
 331       break;
 332     }
 333   } else {
 334 #ifdef _LP64
 335     ShouldNotReachHere();
 336 #else
 337            if (value == 0) { __ fldz();
 338     } else if (value == 1) { __ fld1();
 339     } else                 { ShouldNotReachHere();
 340     }
 341 #endif
 342   }
 343 }
 344 
 345 void TemplateTable::bipush() {
 346   transition(vtos, itos);
 347   __ load_signed_byte(rax, at_bcp(1));
 348 }
 349 
 350 void TemplateTable::sipush() {
 351   transition(vtos, itos);
 352   __ load_unsigned_short(rax, at_bcp(1));
 353   __ bswapl(rax);
 354   __ sarl(rax, 16);
 355 }
 356 
 357 void TemplateTable::ldc(LdcType type) {
 358   transition(vtos, vtos);
 359   Register rarg = NOT_LP64(rcx) LP64_ONLY(c_rarg1);
 360   Label call_ldc, notFloat, notClass, notInt, Done;
 361 
 362   if (is_ldc_wide(type)) {
 363     __ get_unsigned_2_byte_index_at_bcp(rbx, 1);
 364   } else {
 365     __ load_unsigned_byte(rbx, at_bcp(1));
 366   }
 367 
 368   __ get_cpool_and_tags(rcx, rax);
 369   const int base_offset = ConstantPool::header_size() * wordSize;
 370   const int tags_offset = Array<u1>::base_offset_in_bytes();
 371 
 372   // get type
 373   __ movzbl(rdx, Address(rax, rbx, Address::times_1, tags_offset));
 374 
 375   // unresolved class - get the resolved class
 376   __ cmpl(rdx, JVM_CONSTANT_UnresolvedClass);
 377   __ jccb(Assembler::equal, call_ldc);
 378 
 379   // unresolved class in error state - call into runtime to throw the error
 380   // from the first resolution attempt
 381   __ cmpl(rdx, JVM_CONSTANT_UnresolvedClassInError);
 382   __ jccb(Assembler::equal, call_ldc);
 383 
 384   // resolved class - need to call vm to get java mirror of the class
 385   __ cmpl(rdx, JVM_CONSTANT_Class);
 386   __ jcc(Assembler::notEqual, notClass);
 387 
 388   __ bind(call_ldc);
 389 
 390   __ movl(rarg, is_ldc_wide(type) ? 1 : 0);
 391   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), rarg);
 392 
 393   __ push(atos);
 394   __ jmp(Done);
 395 
 396   __ bind(notClass);
 397   __ cmpl(rdx, JVM_CONSTANT_Float);
 398   __ jccb(Assembler::notEqual, notFloat);
 399 
 400   // ftos
 401   __ load_float(Address(rcx, rbx, Address::times_ptr, base_offset));
 402   __ push(ftos);
 403   __ jmp(Done);
 404 
 405   __ bind(notFloat);
 406   __ cmpl(rdx, JVM_CONSTANT_Integer);
 407   __ jccb(Assembler::notEqual, notInt);
 408 
 409   // itos
 410   __ movl(rax, Address(rcx, rbx, Address::times_ptr, base_offset));
 411   __ push(itos);
 412   __ jmp(Done);
 413 
 414   // assume the tag is for condy; if not, the VM runtime will tell us
 415   __ bind(notInt);
 416   condy_helper(Done);
 417 
 418   __ bind(Done);
 419 }
 420 
 421 // Fast path for caching oop constants.
 422 void TemplateTable::fast_aldc(LdcType type) {
 423   transition(vtos, atos);
 424 
 425   Register result = rax;
 426   Register tmp = rdx;
 427   Register rarg = NOT_LP64(rcx) LP64_ONLY(c_rarg1);
 428   int index_size = is_ldc_wide(type) ? sizeof(u2) : sizeof(u1);
 429 
 430   Label resolved;
 431 
 432   // We are resolved if the resolved reference cache entry contains a
 433   // non-null object (String, MethodType, etc.)
 434   assert_different_registers(result, tmp);
 435   __ get_cache_index_at_bcp(tmp, 1, index_size);
 436   __ load_resolved_reference_at_index(result, tmp);
 437   __ testptr(result, result);
 438   __ jcc(Assembler::notZero, resolved);
 439 
 440   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 441 
 442   // first time invocation - must resolve first
 443   __ movl(rarg, (int)bytecode());
 444   __ call_VM(result, entry, rarg);
 445   __ bind(resolved);
 446 
 447   { // Check for the null sentinel.
 448     // If we just called the VM, it already did the mapping for us,
 449     // but it's harmless to retry.
 450     Label notNull;
 451     ExternalAddress null_sentinel((address)Universe::the_null_sentinel_addr());
 452     __ movptr(tmp, null_sentinel);
 453     __ resolve_oop_handle(tmp, rscratch2);
 454     __ cmpoop(tmp, result);
 455     __ jccb(Assembler::notEqual, notNull);
 456     __ xorptr(result, result);  // null object reference
 457     __ bind(notNull);
 458   }
 459 
 460   if (VerifyOops) {
 461     __ verify_oop(result);
 462   }
 463 }
 464 
 465 void TemplateTable::ldc2_w() {
 466   transition(vtos, vtos);
 467   Label notDouble, notLong, Done;
 468   __ get_unsigned_2_byte_index_at_bcp(rbx, 1);
 469 
 470   __ get_cpool_and_tags(rcx, rax);
 471   const int base_offset = ConstantPool::header_size() * wordSize;
 472   const int tags_offset = Array<u1>::base_offset_in_bytes();
 473 
 474   // get type
 475   __ movzbl(rdx, Address(rax, rbx, Address::times_1, tags_offset));
 476   __ cmpl(rdx, JVM_CONSTANT_Double);
 477   __ jccb(Assembler::notEqual, notDouble);
 478 
 479   // dtos
 480   __ load_double(Address(rcx, rbx, Address::times_ptr, base_offset));
 481   __ push(dtos);
 482 
 483   __ jmp(Done);
 484   __ bind(notDouble);
 485   __ cmpl(rdx, JVM_CONSTANT_Long);
 486   __ jccb(Assembler::notEqual, notLong);
 487 
 488   // ltos
 489   __ movptr(rax, Address(rcx, rbx, Address::times_ptr, base_offset + 0 * wordSize));
 490   NOT_LP64(__ movptr(rdx, Address(rcx, rbx, Address::times_ptr, base_offset + 1 * wordSize)));
 491   __ push(ltos);
 492   __ jmp(Done);
 493 
 494   __ bind(notLong);
 495   condy_helper(Done);
 496 
 497   __ bind(Done);
 498 }
 499 
 500 void TemplateTable::condy_helper(Label& Done) {
 501   const Register obj = rax;
 502   const Register off = rbx;
 503   const Register flags = rcx;
 504   const Register rarg = NOT_LP64(rcx) LP64_ONLY(c_rarg1);
 505   __ movl(rarg, (int)bytecode());
 506   call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc), rarg);
 507 #ifndef _LP64
 508   // borrow rdi from locals
 509   __ get_thread(rdi);
 510   __ get_vm_result_2(flags, rdi);
 511   __ restore_locals();
 512 #else
 513   __ get_vm_result_2(flags, r15_thread);
 514 #endif
 515   // VMr = obj = base address to find primitive value to push
 516   // VMr2 = flags = (tos, off) using format of CPCE::_flags
 517   __ movl(off, flags);
 518   __ andl(off, ConstantPoolCacheEntry::field_index_mask);
 519   const Address field(obj, off, Address::times_1, 0*wordSize);
 520 
 521   // What sort of thing are we loading?
 522   __ shrl(flags, ConstantPoolCacheEntry::tos_state_shift);
 523   __ andl(flags, ConstantPoolCacheEntry::tos_state_mask);
 524 
 525   switch (bytecode()) {
 526   case Bytecodes::_ldc:
 527   case Bytecodes::_ldc_w:
 528     {
 529       // tos in (itos, ftos, stos, btos, ctos, ztos)
 530       Label notInt, notFloat, notShort, notByte, notChar, notBool;
 531       __ cmpl(flags, itos);
 532       __ jccb(Assembler::notEqual, notInt);
 533       // itos
 534       __ movl(rax, field);
 535       __ push(itos);
 536       __ jmp(Done);
 537 
 538       __ bind(notInt);
 539       __ cmpl(flags, ftos);
 540       __ jccb(Assembler::notEqual, notFloat);
 541       // ftos
 542       __ load_float(field);
 543       __ push(ftos);
 544       __ jmp(Done);
 545 
 546       __ bind(notFloat);
 547       __ cmpl(flags, stos);
 548       __ jccb(Assembler::notEqual, notShort);
 549       // stos
 550       __ load_signed_short(rax, field);
 551       __ push(stos);
 552       __ jmp(Done);
 553 
 554       __ bind(notShort);
 555       __ cmpl(flags, btos);
 556       __ jccb(Assembler::notEqual, notByte);
 557       // btos
 558       __ load_signed_byte(rax, field);
 559       __ push(btos);
 560       __ jmp(Done);
 561 
 562       __ bind(notByte);
 563       __ cmpl(flags, ctos);
 564       __ jccb(Assembler::notEqual, notChar);
 565       // ctos
 566       __ load_unsigned_short(rax, field);
 567       __ push(ctos);
 568       __ jmp(Done);
 569 
 570       __ bind(notChar);
 571       __ cmpl(flags, ztos);
 572       __ jccb(Assembler::notEqual, notBool);
 573       // ztos
 574       __ load_signed_byte(rax, field);
 575       __ push(ztos);
 576       __ jmp(Done);
 577 
 578       __ bind(notBool);
 579       break;
 580     }
 581 
 582   case Bytecodes::_ldc2_w:
 583     {
 584       Label notLong, notDouble;
 585       __ cmpl(flags, ltos);
 586       __ jccb(Assembler::notEqual, notLong);
 587       // ltos
 588       // Loading high word first because movptr clobbers rax
 589       NOT_LP64(__ movptr(rdx, field.plus_disp(4)));
 590       __ movptr(rax, field);
 591       __ push(ltos);
 592       __ jmp(Done);
 593 
 594       __ bind(notLong);
 595       __ cmpl(flags, dtos);
 596       __ jccb(Assembler::notEqual, notDouble);
 597       // dtos
 598       __ load_double(field);
 599       __ push(dtos);
 600       __ jmp(Done);
 601 
 602       __ bind(notDouble);
 603       break;
 604     }
 605 
 606   default:
 607     ShouldNotReachHere();
 608   }
 609 
 610   __ stop("bad ldc/condy");
 611 }
 612 
 613 void TemplateTable::locals_index(Register reg, int offset) {
 614   __ load_unsigned_byte(reg, at_bcp(offset));
 615   __ negptr(reg);
 616 }
 617 
 618 void TemplateTable::iload() {
 619   iload_internal();
 620 }
 621 
 622 void TemplateTable::nofast_iload() {
 623   iload_internal(may_not_rewrite);
 624 }
 625 
 626 void TemplateTable::iload_internal(RewriteControl rc) {
 627   transition(vtos, itos);
 628   if (RewriteFrequentPairs && rc == may_rewrite) {
 629     Label rewrite, done;
 630     const Register bc = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
 631     LP64_ONLY(assert(rbx != bc, "register damaged"));
 632 
 633     // get next byte
 634     __ load_unsigned_byte(rbx,
 635                           at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 636     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 637     // last two iloads in a pair.  Comparing against fast_iload means that
 638     // the next bytecode is neither an iload or a caload, and therefore
 639     // an iload pair.
 640     __ cmpl(rbx, Bytecodes::_iload);
 641     __ jcc(Assembler::equal, done);
 642 
 643     __ cmpl(rbx, Bytecodes::_fast_iload);
 644     __ movl(bc, Bytecodes::_fast_iload2);
 645 
 646     __ jccb(Assembler::equal, rewrite);
 647 
 648     // if _caload, rewrite to fast_icaload
 649     __ cmpl(rbx, Bytecodes::_caload);
 650     __ movl(bc, Bytecodes::_fast_icaload);
 651     __ jccb(Assembler::equal, rewrite);
 652 
 653     // rewrite so iload doesn't check again.
 654     __ movl(bc, Bytecodes::_fast_iload);
 655 
 656     // rewrite
 657     // bc: fast bytecode
 658     __ bind(rewrite);
 659     patch_bytecode(Bytecodes::_iload, bc, rbx, false);
 660     __ bind(done);
 661   }
 662 
 663   // Get the local value into tos
 664   locals_index(rbx);
 665   __ movl(rax, iaddress(rbx));
 666 }
 667 
 668 void TemplateTable::fast_iload2() {
 669   transition(vtos, itos);
 670   locals_index(rbx);
 671   __ movl(rax, iaddress(rbx));
 672   __ push(itos);
 673   locals_index(rbx, 3);
 674   __ movl(rax, iaddress(rbx));
 675 }
 676 
 677 void TemplateTable::fast_iload() {
 678   transition(vtos, itos);
 679   locals_index(rbx);
 680   __ movl(rax, iaddress(rbx));
 681 }
 682 
 683 void TemplateTable::lload() {
 684   transition(vtos, ltos);
 685   locals_index(rbx);
 686   __ movptr(rax, laddress(rbx));
 687   NOT_LP64(__ movl(rdx, haddress(rbx)));
 688 }
 689 
 690 void TemplateTable::fload() {
 691   transition(vtos, ftos);
 692   locals_index(rbx);
 693   __ load_float(faddress(rbx));
 694 }
 695 
 696 void TemplateTable::dload() {
 697   transition(vtos, dtos);
 698   locals_index(rbx);
 699   __ load_double(daddress(rbx));
 700 }
 701 
 702 void TemplateTable::aload() {
 703   transition(vtos, atos);
 704   locals_index(rbx);
 705   __ movptr(rax, aaddress(rbx));
 706 }
 707 
 708 void TemplateTable::locals_index_wide(Register reg) {
 709   __ load_unsigned_short(reg, at_bcp(2));
 710   __ bswapl(reg);
 711   __ shrl(reg, 16);
 712   __ negptr(reg);
 713 }
 714 
 715 void TemplateTable::wide_iload() {
 716   transition(vtos, itos);
 717   locals_index_wide(rbx);
 718   __ movl(rax, iaddress(rbx));
 719 }
 720 
 721 void TemplateTable::wide_lload() {
 722   transition(vtos, ltos);
 723   locals_index_wide(rbx);
 724   __ movptr(rax, laddress(rbx));
 725   NOT_LP64(__ movl(rdx, haddress(rbx)));
 726 }
 727 
 728 void TemplateTable::wide_fload() {
 729   transition(vtos, ftos);
 730   locals_index_wide(rbx);
 731   __ load_float(faddress(rbx));
 732 }
 733 
 734 void TemplateTable::wide_dload() {
 735   transition(vtos, dtos);
 736   locals_index_wide(rbx);
 737   __ load_double(daddress(rbx));
 738 }
 739 
 740 void TemplateTable::wide_aload() {
 741   transition(vtos, atos);
 742   locals_index_wide(rbx);
 743   __ movptr(rax, aaddress(rbx));
 744 }
 745 
 746 void TemplateTable::index_check(Register array, Register index) {
 747   // Pop ptr into array
 748   __ pop_ptr(array);
 749   index_check_without_pop(array, index);
 750 }
 751 
 752 void TemplateTable::index_check_without_pop(Register array, Register index) {
 753   // destroys rbx
 754   // sign extend index for use by indexed load
 755   __ movl2ptr(index, index);
 756   // check index
 757   __ cmpl(index, Address(array, arrayOopDesc::length_offset_in_bytes()));
 758   if (index != rbx) {
 759     // ??? convention: move aberrant index into rbx for exception message
 760     assert(rbx != array, "different registers");
 761     __ movl(rbx, index);
 762   }
 763   Label skip;
 764   __ jccb(Assembler::below, skip);
 765   // Pass array to create more detailed exceptions.
 766   __ mov(NOT_LP64(rax) LP64_ONLY(c_rarg1), array);
 767   __ jump(ExternalAddress(Interpreter::_throw_ArrayIndexOutOfBoundsException_entry));
 768   __ bind(skip);
 769 }
 770 
 771 void TemplateTable::iaload() {
 772   transition(itos, itos);
 773   // rax: index
 774   // rdx: array
 775   index_check(rdx, rax); // kills rbx
 776   __ access_load_at(T_INT, IN_HEAP | IS_ARRAY, rax,
 777                     Address(rdx, rax, Address::times_4,
 778                             arrayOopDesc::base_offset_in_bytes(T_INT)),
 779                     noreg, noreg);
 780 }
 781 
 782 void TemplateTable::laload() {
 783   transition(itos, ltos);
 784   // rax: index
 785   // rdx: array
 786   index_check(rdx, rax); // kills rbx
 787   NOT_LP64(__ mov(rbx, rax));
 788   // rbx,: index
 789   __ access_load_at(T_LONG, IN_HEAP | IS_ARRAY, noreg /* ltos */,
 790                     Address(rdx, rbx, Address::times_8,
 791                             arrayOopDesc::base_offset_in_bytes(T_LONG)),
 792                     noreg, noreg);
 793 }
 794 
 795 
 796 
 797 void TemplateTable::faload() {
 798   transition(itos, ftos);
 799   // rax: index
 800   // rdx: array
 801   index_check(rdx, rax); // kills rbx
 802   __ access_load_at(T_FLOAT, IN_HEAP | IS_ARRAY, noreg /* ftos */,
 803                     Address(rdx, rax,
 804                             Address::times_4,
 805                             arrayOopDesc::base_offset_in_bytes(T_FLOAT)),
 806                     noreg, noreg);
 807 }
 808 
 809 void TemplateTable::daload() {
 810   transition(itos, dtos);
 811   // rax: index
 812   // rdx: array
 813   index_check(rdx, rax); // kills rbx
 814   __ access_load_at(T_DOUBLE, IN_HEAP | IS_ARRAY, noreg /* dtos */,
 815                     Address(rdx, rax,
 816                             Address::times_8,
 817                             arrayOopDesc::base_offset_in_bytes(T_DOUBLE)),
 818                     noreg, noreg);
 819 }
 820 
 821 void TemplateTable::aaload() {
 822   transition(itos, atos);
 823   // rax: index
 824   // rdx: array
 825   index_check(rdx, rax); // kills rbx
 826   do_oop_load(_masm,
 827               Address(rdx, rax,
 828                       UseCompressedOops ? Address::times_4 : Address::times_ptr,
 829                       arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
 830               rax,
 831               IS_ARRAY);
 832 }
 833 
 834 void TemplateTable::baload() {
 835   transition(itos, itos);
 836   // rax: index
 837   // rdx: array
 838   index_check(rdx, rax); // kills rbx
 839   __ access_load_at(T_BYTE, IN_HEAP | IS_ARRAY, rax,
 840                     Address(rdx, rax, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_BYTE)),
 841                     noreg, noreg);
 842 }
 843 
 844 void TemplateTable::caload() {
 845   transition(itos, itos);
 846   // rax: index
 847   // rdx: array
 848   index_check(rdx, rax); // kills rbx
 849   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, rax,
 850                     Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)),
 851                     noreg, noreg);
 852 }
 853 
 854 // iload followed by caload frequent pair
 855 void TemplateTable::fast_icaload() {
 856   transition(vtos, itos);
 857   // load index out of locals
 858   locals_index(rbx);
 859   __ movl(rax, iaddress(rbx));
 860 
 861   // rax: index
 862   // rdx: array
 863   index_check(rdx, rax); // kills rbx
 864   __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, rax,
 865                     Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)),
 866                     noreg, noreg);
 867 }
 868 
 869 
 870 void TemplateTable::saload() {
 871   transition(itos, itos);
 872   // rax: index
 873   // rdx: array
 874   index_check(rdx, rax); // kills rbx
 875   __ access_load_at(T_SHORT, IN_HEAP | IS_ARRAY, rax,
 876                     Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_SHORT)),
 877                     noreg, noreg);
 878 }
 879 
 880 void TemplateTable::iload(int n) {
 881   transition(vtos, itos);
 882   __ movl(rax, iaddress(n));
 883 }
 884 
 885 void TemplateTable::lload(int n) {
 886   transition(vtos, ltos);
 887   __ movptr(rax, laddress(n));
 888   NOT_LP64(__ movptr(rdx, haddress(n)));
 889 }
 890 
 891 void TemplateTable::fload(int n) {
 892   transition(vtos, ftos);
 893   __ load_float(faddress(n));
 894 }
 895 
 896 void TemplateTable::dload(int n) {
 897   transition(vtos, dtos);
 898   __ load_double(daddress(n));
 899 }
 900 
 901 void TemplateTable::aload(int n) {
 902   transition(vtos, atos);
 903   __ movptr(rax, aaddress(n));
 904 }
 905 
 906 void TemplateTable::aload_0() {
 907   aload_0_internal();
 908 }
 909 
 910 void TemplateTable::nofast_aload_0() {
 911   aload_0_internal(may_not_rewrite);
 912 }
 913 
 914 void TemplateTable::aload_0_internal(RewriteControl rc) {
 915   transition(vtos, atos);
 916   // According to bytecode histograms, the pairs:
 917   //
 918   // _aload_0, _fast_igetfield
 919   // _aload_0, _fast_agetfield
 920   // _aload_0, _fast_fgetfield
 921   //
 922   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 923   // _aload_0 bytecode checks if the next bytecode is either
 924   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 925   // rewrites the current bytecode into a pair bytecode; otherwise it
 926   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 927   // the pair check anymore.
 928   //
 929   // Note: If the next bytecode is _getfield, the rewrite must be
 930   //       delayed, otherwise we may miss an opportunity for a pair.
 931   //
 932   // Also rewrite frequent pairs
 933   //   aload_0, aload_1
 934   //   aload_0, iload_1
 935   // These bytecodes with a small amount of code are most profitable
 936   // to rewrite
 937   if (RewriteFrequentPairs && rc == may_rewrite) {
 938     Label rewrite, done;
 939 
 940     const Register bc = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
 941     LP64_ONLY(assert(rbx != bc, "register damaged"));
 942 
 943     // get next byte
 944     __ load_unsigned_byte(rbx, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 945 
 946     // if _getfield then wait with rewrite
 947     __ cmpl(rbx, Bytecodes::_getfield);
 948     __ jcc(Assembler::equal, done);
 949 
 950     // if _igetfield then rewrite to _fast_iaccess_0
 951     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 952     __ cmpl(rbx, Bytecodes::_fast_igetfield);
 953     __ movl(bc, Bytecodes::_fast_iaccess_0);
 954     __ jccb(Assembler::equal, rewrite);
 955 
 956     // if _agetfield then rewrite to _fast_aaccess_0
 957     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 958     __ cmpl(rbx, Bytecodes::_fast_agetfield);
 959     __ movl(bc, Bytecodes::_fast_aaccess_0);
 960     __ jccb(Assembler::equal, rewrite);
 961 
 962     // if _fgetfield then rewrite to _fast_faccess_0
 963     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 964     __ cmpl(rbx, Bytecodes::_fast_fgetfield);
 965     __ movl(bc, Bytecodes::_fast_faccess_0);
 966     __ jccb(Assembler::equal, rewrite);
 967 
 968     // else rewrite to _fast_aload0
 969     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 970     __ movl(bc, Bytecodes::_fast_aload_0);
 971 
 972     // rewrite
 973     // bc: fast bytecode
 974     __ bind(rewrite);
 975     patch_bytecode(Bytecodes::_aload_0, bc, rbx, false);
 976 
 977     __ bind(done);
 978   }
 979 
 980   // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
 981   aload(0);
 982 }
 983 
 984 void TemplateTable::istore() {
 985   transition(itos, vtos);
 986   locals_index(rbx);
 987   __ movl(iaddress(rbx), rax);
 988 }
 989 
 990 
 991 void TemplateTable::lstore() {
 992   transition(ltos, vtos);
 993   locals_index(rbx);
 994   __ movptr(laddress(rbx), rax);
 995   NOT_LP64(__ movptr(haddress(rbx), rdx));
 996 }
 997 
 998 void TemplateTable::fstore() {
 999   transition(ftos, vtos);
1000   locals_index(rbx);
1001   __ store_float(faddress(rbx));
1002 }
1003 
1004 void TemplateTable::dstore() {
1005   transition(dtos, vtos);
1006   locals_index(rbx);
1007   __ store_double(daddress(rbx));
1008 }
1009 
1010 void TemplateTable::astore() {
1011   transition(vtos, vtos);
1012   __ pop_ptr(rax);
1013   locals_index(rbx);
1014   __ movptr(aaddress(rbx), rax);
1015 }
1016 
1017 void TemplateTable::wide_istore() {
1018   transition(vtos, vtos);
1019   __ pop_i();
1020   locals_index_wide(rbx);
1021   __ movl(iaddress(rbx), rax);
1022 }
1023 
1024 void TemplateTable::wide_lstore() {
1025   transition(vtos, vtos);
1026   NOT_LP64(__ pop_l(rax, rdx));
1027   LP64_ONLY(__ pop_l());
1028   locals_index_wide(rbx);
1029   __ movptr(laddress(rbx), rax);
1030   NOT_LP64(__ movl(haddress(rbx), rdx));
1031 }
1032 
1033 void TemplateTable::wide_fstore() {
1034 #ifdef _LP64
1035   transition(vtos, vtos);
1036   __ pop_f(xmm0);
1037   locals_index_wide(rbx);
1038   __ movflt(faddress(rbx), xmm0);
1039 #else
1040   wide_istore();
1041 #endif
1042 }
1043 
1044 void TemplateTable::wide_dstore() {
1045 #ifdef _LP64
1046   transition(vtos, vtos);
1047   __ pop_d(xmm0);
1048   locals_index_wide(rbx);
1049   __ movdbl(daddress(rbx), xmm0);
1050 #else
1051   wide_lstore();
1052 #endif
1053 }
1054 
1055 void TemplateTable::wide_astore() {
1056   transition(vtos, vtos);
1057   __ pop_ptr(rax);
1058   locals_index_wide(rbx);
1059   __ movptr(aaddress(rbx), rax);
1060 }
1061 
1062 void TemplateTable::iastore() {
1063   transition(itos, vtos);
1064   __ pop_i(rbx);
1065   // rax: value
1066   // rbx: index
1067   // rdx: array
1068   index_check(rdx, rbx); // prefer index in rbx
1069   __ access_store_at(T_INT, IN_HEAP | IS_ARRAY,
1070                      Address(rdx, rbx, Address::times_4,
1071                              arrayOopDesc::base_offset_in_bytes(T_INT)),
1072                      rax, noreg, noreg, noreg);
1073 }
1074 
1075 void TemplateTable::lastore() {
1076   transition(ltos, vtos);
1077   __ pop_i(rbx);
1078   // rax,: low(value)
1079   // rcx: array
1080   // rdx: high(value)
1081   index_check(rcx, rbx);  // prefer index in rbx,
1082   // rbx,: index
1083   __ access_store_at(T_LONG, IN_HEAP | IS_ARRAY,
1084                      Address(rcx, rbx, Address::times_8,
1085                              arrayOopDesc::base_offset_in_bytes(T_LONG)),
1086                      noreg /* ltos */, noreg, noreg, noreg);
1087 }
1088 
1089 
1090 void TemplateTable::fastore() {
1091   transition(ftos, vtos);
1092   __ pop_i(rbx);
1093   // value is in UseSSE >= 1 ? xmm0 : ST(0)
1094   // rbx:  index
1095   // rdx:  array
1096   index_check(rdx, rbx); // prefer index in rbx
1097   __ access_store_at(T_FLOAT, IN_HEAP | IS_ARRAY,
1098                      Address(rdx, rbx, Address::times_4,
1099                              arrayOopDesc::base_offset_in_bytes(T_FLOAT)),
1100                      noreg /* ftos */, noreg, noreg, noreg);
1101 }
1102 
1103 void TemplateTable::dastore() {
1104   transition(dtos, vtos);
1105   __ pop_i(rbx);
1106   // value is in UseSSE >= 2 ? xmm0 : ST(0)
1107   // rbx:  index
1108   // rdx:  array
1109   index_check(rdx, rbx); // prefer index in rbx
1110   __ access_store_at(T_DOUBLE, IN_HEAP | IS_ARRAY,
1111                      Address(rdx, rbx, Address::times_8,
1112                              arrayOopDesc::base_offset_in_bytes(T_DOUBLE)),
1113                      noreg /* dtos */, noreg, noreg, noreg);
1114 }
1115 
1116 void TemplateTable::aastore() {
1117   Label is_null, ok_is_subtype, done;
1118   transition(vtos, vtos);
1119   // stack: ..., array, index, value
1120   __ movptr(rax, at_tos());    // value
1121   __ movl(rcx, at_tos_p1()); // index
1122   __ movptr(rdx, at_tos_p2()); // array
1123 
1124   Address element_address(rdx, rcx,
1125                           UseCompressedOops? Address::times_4 : Address::times_ptr,
1126                           arrayOopDesc::base_offset_in_bytes(T_OBJECT));
1127 
1128   index_check_without_pop(rdx, rcx);     // kills rbx
1129   __ testptr(rax, rax);
1130   __ jcc(Assembler::zero, is_null);
1131 
1132   // Move subklass into rbx
1133   __ load_klass(rbx, rax, rscratch1);
1134   // Move superklass into rax
1135   __ load_klass(rax, rdx, rscratch1);
1136   __ movptr(rax, Address(rax,
1137                          ObjArrayKlass::element_klass_offset()));
1138 
1139   // Generate subtype check.  Blows rcx, rdi
1140   // Superklass in rax.  Subklass in rbx.
1141   __ gen_subtype_check(rbx, ok_is_subtype);
1142 
1143   // Come here on failure
1144   // object is at TOS
1145   __ jump(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
1146 
1147   // Come here on success
1148   __ bind(ok_is_subtype);
1149 
1150   // Get the value we will store
1151   __ movptr(rax, at_tos());
1152   __ movl(rcx, at_tos_p1()); // index
1153   // Now store using the appropriate barrier
1154   do_oop_store(_masm, element_address, rax, IS_ARRAY);
1155   __ jmp(done);
1156 
1157   // Have a null in rax, rdx=array, ecx=index.  Store null at ary[idx]
1158   __ bind(is_null);
1159   __ profile_null_seen(rbx);
1160 
1161   // Store a null
1162   do_oop_store(_masm, element_address, noreg, IS_ARRAY);
1163 
1164   // Pop stack arguments
1165   __ bind(done);
1166   __ addptr(rsp, 3 * Interpreter::stackElementSize);
1167 }
1168 
1169 void TemplateTable::bastore() {
1170   transition(itos, vtos);
1171   __ pop_i(rbx);
1172   // rax: value
1173   // rbx: index
1174   // rdx: array
1175   index_check(rdx, rbx); // prefer index in rbx
1176   // Need to check whether array is boolean or byte
1177   // since both types share the bastore bytecode.
1178   __ load_klass(rcx, rdx, rscratch1);
1179   __ movl(rcx, Address(rcx, Klass::layout_helper_offset()));
1180   int diffbit = Klass::layout_helper_boolean_diffbit();
1181   __ testl(rcx, diffbit);
1182   Label L_skip;
1183   __ jccb(Assembler::zero, L_skip);
1184   __ andl(rax, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1185   __ bind(L_skip);
1186   __ access_store_at(T_BYTE, IN_HEAP | IS_ARRAY,
1187                      Address(rdx, rbx,Address::times_1,
1188                              arrayOopDesc::base_offset_in_bytes(T_BYTE)),
1189                      rax, noreg, noreg, noreg);
1190 }
1191 
1192 void TemplateTable::castore() {
1193   transition(itos, vtos);
1194   __ pop_i(rbx);
1195   // rax: value
1196   // rbx: index
1197   // rdx: array
1198   index_check(rdx, rbx);  // prefer index in rbx
1199   __ access_store_at(T_CHAR, IN_HEAP | IS_ARRAY,
1200                      Address(rdx, rbx, Address::times_2,
1201                              arrayOopDesc::base_offset_in_bytes(T_CHAR)),
1202                      rax, noreg, noreg, noreg);
1203 }
1204 
1205 
1206 void TemplateTable::sastore() {
1207   castore();
1208 }
1209 
1210 void TemplateTable::istore(int n) {
1211   transition(itos, vtos);
1212   __ movl(iaddress(n), rax);
1213 }
1214 
1215 void TemplateTable::lstore(int n) {
1216   transition(ltos, vtos);
1217   __ movptr(laddress(n), rax);
1218   NOT_LP64(__ movptr(haddress(n), rdx));
1219 }
1220 
1221 void TemplateTable::fstore(int n) {
1222   transition(ftos, vtos);
1223   __ store_float(faddress(n));
1224 }
1225 
1226 void TemplateTable::dstore(int n) {
1227   transition(dtos, vtos);
1228   __ store_double(daddress(n));
1229 }
1230 
1231 
1232 void TemplateTable::astore(int n) {
1233   transition(vtos, vtos);
1234   __ pop_ptr(rax);
1235   __ movptr(aaddress(n), rax);
1236 }
1237 
1238 void TemplateTable::pop() {
1239   transition(vtos, vtos);
1240   __ addptr(rsp, Interpreter::stackElementSize);
1241 }
1242 
1243 void TemplateTable::pop2() {
1244   transition(vtos, vtos);
1245   __ addptr(rsp, 2 * Interpreter::stackElementSize);
1246 }
1247 
1248 
1249 void TemplateTable::dup() {
1250   transition(vtos, vtos);
1251   __ load_ptr(0, rax);
1252   __ push_ptr(rax);
1253   // stack: ..., a, a
1254 }
1255 
1256 void TemplateTable::dup_x1() {
1257   transition(vtos, vtos);
1258   // stack: ..., a, b
1259   __ load_ptr( 0, rax);  // load b
1260   __ load_ptr( 1, rcx);  // load a
1261   __ store_ptr(1, rax);  // store b
1262   __ store_ptr(0, rcx);  // store a
1263   __ push_ptr(rax);      // push b
1264   // stack: ..., b, a, b
1265 }
1266 
1267 void TemplateTable::dup_x2() {
1268   transition(vtos, vtos);
1269   // stack: ..., a, b, c
1270   __ load_ptr( 0, rax);  // load c
1271   __ load_ptr( 2, rcx);  // load a
1272   __ store_ptr(2, rax);  // store c in a
1273   __ push_ptr(rax);      // push c
1274   // stack: ..., c, b, c, c
1275   __ load_ptr( 2, rax);  // load b
1276   __ store_ptr(2, rcx);  // store a in b
1277   // stack: ..., c, a, c, c
1278   __ store_ptr(1, rax);  // store b in c
1279   // stack: ..., c, a, b, c
1280 }
1281 
1282 void TemplateTable::dup2() {
1283   transition(vtos, vtos);
1284   // stack: ..., a, b
1285   __ load_ptr(1, rax);  // load a
1286   __ push_ptr(rax);     // push a
1287   __ load_ptr(1, rax);  // load b
1288   __ push_ptr(rax);     // push b
1289   // stack: ..., a, b, a, b
1290 }
1291 
1292 
1293 void TemplateTable::dup2_x1() {
1294   transition(vtos, vtos);
1295   // stack: ..., a, b, c
1296   __ load_ptr( 0, rcx);  // load c
1297   __ load_ptr( 1, rax);  // load b
1298   __ push_ptr(rax);      // push b
1299   __ push_ptr(rcx);      // push c
1300   // stack: ..., a, b, c, b, c
1301   __ store_ptr(3, rcx);  // store c in b
1302   // stack: ..., a, c, c, b, c
1303   __ load_ptr( 4, rcx);  // load a
1304   __ store_ptr(2, rcx);  // store a in 2nd c
1305   // stack: ..., a, c, a, b, c
1306   __ store_ptr(4, rax);  // store b in a
1307   // stack: ..., b, c, a, b, c
1308 }
1309 
1310 void TemplateTable::dup2_x2() {
1311   transition(vtos, vtos);
1312   // stack: ..., a, b, c, d
1313   __ load_ptr( 0, rcx);  // load d
1314   __ load_ptr( 1, rax);  // load c
1315   __ push_ptr(rax);      // push c
1316   __ push_ptr(rcx);      // push d
1317   // stack: ..., a, b, c, d, c, d
1318   __ load_ptr( 4, rax);  // load b
1319   __ store_ptr(2, rax);  // store b in d
1320   __ store_ptr(4, rcx);  // store d in b
1321   // stack: ..., a, d, c, b, c, d
1322   __ load_ptr( 5, rcx);  // load a
1323   __ load_ptr( 3, rax);  // load c
1324   __ store_ptr(3, rcx);  // store a in c
1325   __ store_ptr(5, rax);  // store c in a
1326   // stack: ..., c, d, a, b, c, d
1327 }
1328 
1329 void TemplateTable::swap() {
1330   transition(vtos, vtos);
1331   // stack: ..., a, b
1332   __ load_ptr( 1, rcx);  // load a
1333   __ load_ptr( 0, rax);  // load b
1334   __ store_ptr(0, rcx);  // store a in b
1335   __ store_ptr(1, rax);  // store b in a
1336   // stack: ..., b, a
1337 }
1338 
1339 void TemplateTable::iop2(Operation op) {
1340   transition(itos, itos);
1341   switch (op) {
1342   case add  :                    __ pop_i(rdx); __ addl (rax, rdx); break;
1343   case sub  : __ movl(rdx, rax); __ pop_i(rax); __ subl (rax, rdx); break;
1344   case mul  :                    __ pop_i(rdx); __ imull(rax, rdx); break;
1345   case _and :                    __ pop_i(rdx); __ andl (rax, rdx); break;
1346   case _or  :                    __ pop_i(rdx); __ orl  (rax, rdx); break;
1347   case _xor :                    __ pop_i(rdx); __ xorl (rax, rdx); break;
1348   case shl  : __ movl(rcx, rax); __ pop_i(rax); __ shll (rax);      break;
1349   case shr  : __ movl(rcx, rax); __ pop_i(rax); __ sarl (rax);      break;
1350   case ushr : __ movl(rcx, rax); __ pop_i(rax); __ shrl (rax);      break;
1351   default   : ShouldNotReachHere();
1352   }
1353 }
1354 
1355 void TemplateTable::lop2(Operation op) {
1356   transition(ltos, ltos);
1357 #ifdef _LP64
1358   switch (op) {
1359   case add  :                    __ pop_l(rdx); __ addptr(rax, rdx); break;
1360   case sub  : __ mov(rdx, rax);  __ pop_l(rax); __ subptr(rax, rdx); break;
1361   case _and :                    __ pop_l(rdx); __ andptr(rax, rdx); break;
1362   case _or  :                    __ pop_l(rdx); __ orptr (rax, rdx); break;
1363   case _xor :                    __ pop_l(rdx); __ xorptr(rax, rdx); break;
1364   default   : ShouldNotReachHere();
1365   }
1366 #else
1367   __ pop_l(rbx, rcx);
1368   switch (op) {
1369     case add  : __ addl(rax, rbx); __ adcl(rdx, rcx); break;
1370     case sub  : __ subl(rbx, rax); __ sbbl(rcx, rdx);
1371                 __ mov (rax, rbx); __ mov (rdx, rcx); break;
1372     case _and : __ andl(rax, rbx); __ andl(rdx, rcx); break;
1373     case _or  : __ orl (rax, rbx); __ orl (rdx, rcx); break;
1374     case _xor : __ xorl(rax, rbx); __ xorl(rdx, rcx); break;
1375     default   : ShouldNotReachHere();
1376   }
1377 #endif
1378 }
1379 
1380 void TemplateTable::idiv() {
1381   transition(itos, itos);
1382   __ movl(rcx, rax);
1383   __ pop_i(rax);
1384   // Note: could xor rax and ecx and compare with (-1 ^ min_int). If
1385   //       they are not equal, one could do a normal division (no correction
1386   //       needed), which may speed up this implementation for the common case.
1387   //       (see also JVM spec., p.243 & p.271)
1388   __ corrected_idivl(rcx);
1389 }
1390 
1391 void TemplateTable::irem() {
1392   transition(itos, itos);
1393   __ movl(rcx, rax);
1394   __ pop_i(rax);
1395   // Note: could xor rax and ecx and compare with (-1 ^ min_int). If
1396   //       they are not equal, one could do a normal division (no correction
1397   //       needed), which may speed up this implementation for the common case.
1398   //       (see also JVM spec., p.243 & p.271)
1399   __ corrected_idivl(rcx);
1400   __ movl(rax, rdx);
1401 }
1402 
1403 void TemplateTable::lmul() {
1404   transition(ltos, ltos);
1405 #ifdef _LP64
1406   __ pop_l(rdx);
1407   __ imulq(rax, rdx);
1408 #else
1409   __ pop_l(rbx, rcx);
1410   __ push(rcx); __ push(rbx);
1411   __ push(rdx); __ push(rax);
1412   __ lmul(2 * wordSize, 0);
1413   __ addptr(rsp, 4 * wordSize);  // take off temporaries
1414 #endif
1415 }
1416 
1417 void TemplateTable::ldiv() {
1418   transition(ltos, ltos);
1419 #ifdef _LP64
1420   __ mov(rcx, rax);
1421   __ pop_l(rax);
1422   // generate explicit div0 check
1423   __ testq(rcx, rcx);
1424   __ jump_cc(Assembler::zero,
1425              ExternalAddress(Interpreter::_throw_ArithmeticException_entry));
1426   // Note: could xor rax and rcx and compare with (-1 ^ min_int). If
1427   //       they are not equal, one could do a normal division (no correction
1428   //       needed), which may speed up this implementation for the common case.
1429   //       (see also JVM spec., p.243 & p.271)
1430   __ corrected_idivq(rcx); // kills rbx
1431 #else
1432   __ pop_l(rbx, rcx);
1433   __ push(rcx); __ push(rbx);
1434   __ push(rdx); __ push(rax);
1435   // check if y = 0
1436   __ orl(rax, rdx);
1437   __ jump_cc(Assembler::zero,
1438              ExternalAddress(Interpreter::_throw_ArithmeticException_entry));
1439   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::ldiv));
1440   __ addptr(rsp, 4 * wordSize);  // take off temporaries
1441 #endif
1442 }
1443 
1444 void TemplateTable::lrem() {
1445   transition(ltos, ltos);
1446 #ifdef _LP64
1447   __ mov(rcx, rax);
1448   __ pop_l(rax);
1449   __ testq(rcx, rcx);
1450   __ jump_cc(Assembler::zero,
1451              ExternalAddress(Interpreter::_throw_ArithmeticException_entry));
1452   // Note: could xor rax and rcx and compare with (-1 ^ min_int). If
1453   //       they are not equal, one could do a normal division (no correction
1454   //       needed), which may speed up this implementation for the common case.
1455   //       (see also JVM spec., p.243 & p.271)
1456   __ corrected_idivq(rcx); // kills rbx
1457   __ mov(rax, rdx);
1458 #else
1459   __ pop_l(rbx, rcx);
1460   __ push(rcx); __ push(rbx);
1461   __ push(rdx); __ push(rax);
1462   // check if y = 0
1463   __ orl(rax, rdx);
1464   __ jump_cc(Assembler::zero,
1465              ExternalAddress(Interpreter::_throw_ArithmeticException_entry));
1466   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::lrem));
1467   __ addptr(rsp, 4 * wordSize);
1468 #endif
1469 }
1470 
1471 void TemplateTable::lshl() {
1472   transition(itos, ltos);
1473   __ movl(rcx, rax);                             // get shift count
1474   #ifdef _LP64
1475   __ pop_l(rax);                                 // get shift value
1476   __ shlq(rax);
1477 #else
1478   __ pop_l(rax, rdx);                            // get shift value
1479   __ lshl(rdx, rax);
1480 #endif
1481 }
1482 
1483 void TemplateTable::lshr() {
1484 #ifdef _LP64
1485   transition(itos, ltos);
1486   __ movl(rcx, rax);                             // get shift count
1487   __ pop_l(rax);                                 // get shift value
1488   __ sarq(rax);
1489 #else
1490   transition(itos, ltos);
1491   __ mov(rcx, rax);                              // get shift count
1492   __ pop_l(rax, rdx);                            // get shift value
1493   __ lshr(rdx, rax, true);
1494 #endif
1495 }
1496 
1497 void TemplateTable::lushr() {
1498   transition(itos, ltos);
1499 #ifdef _LP64
1500   __ movl(rcx, rax);                             // get shift count
1501   __ pop_l(rax);                                 // get shift value
1502   __ shrq(rax);
1503 #else
1504   __ mov(rcx, rax);                              // get shift count
1505   __ pop_l(rax, rdx);                            // get shift value
1506   __ lshr(rdx, rax);
1507 #endif
1508 }
1509 
1510 void TemplateTable::fop2(Operation op) {
1511   transition(ftos, ftos);
1512 
1513   if (UseSSE >= 1) {
1514     switch (op) {
1515     case add:
1516       __ addss(xmm0, at_rsp());
1517       __ addptr(rsp, Interpreter::stackElementSize);
1518       break;
1519     case sub:
1520       __ movflt(xmm1, xmm0);
1521       __ pop_f(xmm0);
1522       __ subss(xmm0, xmm1);
1523       break;
1524     case mul:
1525       __ mulss(xmm0, at_rsp());
1526       __ addptr(rsp, Interpreter::stackElementSize);
1527       break;
1528     case div:
1529       __ movflt(xmm1, xmm0);
1530       __ pop_f(xmm0);
1531       __ divss(xmm0, xmm1);
1532       break;
1533     case rem:
1534       // On x86_64 platforms the SharedRuntime::frem method is called to perform the
1535       // modulo operation. The frem method calls the function
1536       // double fmod(double x, double y) in math.h. The documentation of fmod states:
1537       // "If x or y is a NaN, a NaN is returned." without specifying what type of NaN
1538       // (signalling or quiet) is returned.
1539       //
1540       // On x86_32 platforms the FPU is used to perform the modulo operation. The
1541       // reason is that on 32-bit Windows the sign of modulo operations diverges from
1542       // what is considered the standard (e.g., -0.0f % -3.14f is 0.0f (and not -0.0f).
1543       // The fprem instruction used on x86_32 is functionally equivalent to
1544       // SharedRuntime::frem in that it returns a NaN.
1545 #ifdef _LP64
1546       __ movflt(xmm1, xmm0);
1547       __ pop_f(xmm0);
1548       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem), 2);
1549 #else // !_LP64
1550       __ push_f(xmm0);
1551       __ pop_f();
1552       __ fld_s(at_rsp());
1553       __ fremr(rax);
1554       __ f2ieee();
1555       __ pop(rax);  // pop second operand off the stack
1556       __ push_f();
1557       __ pop_f(xmm0);
1558 #endif // _LP64
1559       break;
1560     default:
1561       ShouldNotReachHere();
1562       break;
1563     }
1564   } else {
1565 #ifdef _LP64
1566     ShouldNotReachHere();
1567 #else // !_LP64
1568     switch (op) {
1569     case add: __ fadd_s (at_rsp());                break;
1570     case sub: __ fsubr_s(at_rsp());                break;
1571     case mul: __ fmul_s (at_rsp());                break;
1572     case div: __ fdivr_s(at_rsp());                break;
1573     case rem: __ fld_s  (at_rsp()); __ fremr(rax); break;
1574     default : ShouldNotReachHere();
1575     }
1576     __ f2ieee();
1577     __ pop(rax);  // pop second operand off the stack
1578 #endif // _LP64
1579   }
1580 }
1581 
1582 void TemplateTable::dop2(Operation op) {
1583   transition(dtos, dtos);
1584   if (UseSSE >= 2) {
1585     switch (op) {
1586     case add:
1587       __ addsd(xmm0, at_rsp());
1588       __ addptr(rsp, 2 * Interpreter::stackElementSize);
1589       break;
1590     case sub:
1591       __ movdbl(xmm1, xmm0);
1592       __ pop_d(xmm0);
1593       __ subsd(xmm0, xmm1);
1594       break;
1595     case mul:
1596       __ mulsd(xmm0, at_rsp());
1597       __ addptr(rsp, 2 * Interpreter::stackElementSize);
1598       break;
1599     case div:
1600       __ movdbl(xmm1, xmm0);
1601       __ pop_d(xmm0);
1602       __ divsd(xmm0, xmm1);
1603       break;
1604     case rem:
1605       // Similar to fop2(), the modulo operation is performed using the
1606       // SharedRuntime::drem method (on x86_64 platforms) or using the
1607       // FPU (on x86_32 platforms) for the same reasons as mentioned in fop2().
1608 #ifdef _LP64
1609       __ movdbl(xmm1, xmm0);
1610       __ pop_d(xmm0);
1611       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem), 2);
1612 #else // !_LP64
1613       __ push_d(xmm0);
1614       __ pop_d();
1615       __ fld_d(at_rsp());
1616       __ fremr(rax);
1617       __ d2ieee();
1618       __ pop(rax);
1619       __ pop(rdx);
1620       __ push_d();
1621       __ pop_d(xmm0);
1622 #endif // _LP64
1623       break;
1624     default:
1625       ShouldNotReachHere();
1626       break;
1627     }
1628   } else {
1629 #ifdef _LP64
1630     ShouldNotReachHere();
1631 #else // !_LP64
1632     switch (op) {
1633     case add: __ fadd_d (at_rsp());                break;
1634     case sub: __ fsubr_d(at_rsp());                break;
1635     case mul: {
1636       // strict semantics
1637       __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias1()));
1638       __ fmulp();
1639       __ fmul_d (at_rsp());
1640       __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias2()));
1641       __ fmulp();
1642       break;
1643     }
1644     case div: {
1645       // strict semantics
1646       __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias1()));
1647       __ fmul_d (at_rsp());
1648       __ fdivrp();
1649       __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias2()));
1650       __ fmulp();
1651       break;
1652     }
1653     case rem: __ fld_d  (at_rsp()); __ fremr(rax); break;
1654     default : ShouldNotReachHere();
1655     }
1656     __ d2ieee();
1657     // Pop double precision number from rsp.
1658     __ pop(rax);
1659     __ pop(rdx);
1660 #endif // _LP64
1661   }
1662 }
1663 
1664 void TemplateTable::ineg() {
1665   transition(itos, itos);
1666   __ negl(rax);
1667 }
1668 
1669 void TemplateTable::lneg() {
1670   transition(ltos, ltos);
1671   LP64_ONLY(__ negq(rax));
1672   NOT_LP64(__ lneg(rdx, rax));
1673 }
1674 
1675 // Note: 'double' and 'long long' have 32-bits alignment on x86.
1676 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
1677   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
1678   // of 128-bits operands for SSE instructions.
1679   jlong *operand = (jlong*)(((intptr_t)adr)&((intptr_t)(~0xF)));
1680   // Store the value to a 128-bits operand.
1681   operand[0] = lo;
1682   operand[1] = hi;
1683   return operand;
1684 }
1685 
1686 // Buffer for 128-bits masks used by SSE instructions.
1687 static jlong float_signflip_pool[2*2];
1688 static jlong double_signflip_pool[2*2];
1689 
1690 void TemplateTable::fneg() {
1691   transition(ftos, ftos);
1692   if (UseSSE >= 1) {
1693     static jlong *float_signflip  = double_quadword(&float_signflip_pool[1],  CONST64(0x8000000080000000),  CONST64(0x8000000080000000));
1694     __ xorps(xmm0, ExternalAddress((address) float_signflip), rscratch1);
1695   } else {
1696     LP64_ONLY(ShouldNotReachHere());
1697     NOT_LP64(__ fchs());
1698   }
1699 }
1700 
1701 void TemplateTable::dneg() {
1702   transition(dtos, dtos);
1703   if (UseSSE >= 2) {
1704     static jlong *double_signflip =
1705       double_quadword(&double_signflip_pool[1], CONST64(0x8000000000000000), CONST64(0x8000000000000000));
1706     __ xorpd(xmm0, ExternalAddress((address) double_signflip), rscratch1);
1707   } else {
1708 #ifdef _LP64
1709     ShouldNotReachHere();
1710 #else
1711     __ fchs();
1712 #endif
1713   }
1714 }
1715 
1716 void TemplateTable::iinc() {
1717   transition(vtos, vtos);
1718   __ load_signed_byte(rdx, at_bcp(2)); // get constant
1719   locals_index(rbx);
1720   __ addl(iaddress(rbx), rdx);
1721 }
1722 
1723 void TemplateTable::wide_iinc() {
1724   transition(vtos, vtos);
1725   __ movl(rdx, at_bcp(4)); // get constant
1726   locals_index_wide(rbx);
1727   __ bswapl(rdx); // swap bytes & sign-extend constant
1728   __ sarl(rdx, 16);
1729   __ addl(iaddress(rbx), rdx);
1730   // Note: should probably use only one movl to get both
1731   //       the index and the constant -> fix this
1732 }
1733 
1734 void TemplateTable::convert() {
1735 #ifdef _LP64
1736   // Checking
1737 #ifdef ASSERT
1738   {
1739     TosState tos_in  = ilgl;
1740     TosState tos_out = ilgl;
1741     switch (bytecode()) {
1742     case Bytecodes::_i2l: // fall through
1743     case Bytecodes::_i2f: // fall through
1744     case Bytecodes::_i2d: // fall through
1745     case Bytecodes::_i2b: // fall through
1746     case Bytecodes::_i2c: // fall through
1747     case Bytecodes::_i2s: tos_in = itos; break;
1748     case Bytecodes::_l2i: // fall through
1749     case Bytecodes::_l2f: // fall through
1750     case Bytecodes::_l2d: tos_in = ltos; break;
1751     case Bytecodes::_f2i: // fall through
1752     case Bytecodes::_f2l: // fall through
1753     case Bytecodes::_f2d: tos_in = ftos; break;
1754     case Bytecodes::_d2i: // fall through
1755     case Bytecodes::_d2l: // fall through
1756     case Bytecodes::_d2f: tos_in = dtos; break;
1757     default             : ShouldNotReachHere();
1758     }
1759     switch (bytecode()) {
1760     case Bytecodes::_l2i: // fall through
1761     case Bytecodes::_f2i: // fall through
1762     case Bytecodes::_d2i: // fall through
1763     case Bytecodes::_i2b: // fall through
1764     case Bytecodes::_i2c: // fall through
1765     case Bytecodes::_i2s: tos_out = itos; break;
1766     case Bytecodes::_i2l: // fall through
1767     case Bytecodes::_f2l: // fall through
1768     case Bytecodes::_d2l: tos_out = ltos; break;
1769     case Bytecodes::_i2f: // fall through
1770     case Bytecodes::_l2f: // fall through
1771     case Bytecodes::_d2f: tos_out = ftos; break;
1772     case Bytecodes::_i2d: // fall through
1773     case Bytecodes::_l2d: // fall through
1774     case Bytecodes::_f2d: tos_out = dtos; break;
1775     default             : ShouldNotReachHere();
1776     }
1777     transition(tos_in, tos_out);
1778   }
1779 #endif // ASSERT
1780 
1781   static const int64_t is_nan = 0x8000000000000000L;
1782 
1783   // Conversion
1784   switch (bytecode()) {
1785   case Bytecodes::_i2l:
1786     __ movslq(rax, rax);
1787     break;
1788   case Bytecodes::_i2f:
1789     __ cvtsi2ssl(xmm0, rax);
1790     break;
1791   case Bytecodes::_i2d:
1792     __ cvtsi2sdl(xmm0, rax);
1793     break;
1794   case Bytecodes::_i2b:
1795     __ movsbl(rax, rax);
1796     break;
1797   case Bytecodes::_i2c:
1798     __ movzwl(rax, rax);
1799     break;
1800   case Bytecodes::_i2s:
1801     __ movswl(rax, rax);
1802     break;
1803   case Bytecodes::_l2i:
1804     __ movl(rax, rax);
1805     break;
1806   case Bytecodes::_l2f:
1807     __ cvtsi2ssq(xmm0, rax);
1808     break;
1809   case Bytecodes::_l2d:
1810     __ cvtsi2sdq(xmm0, rax);
1811     break;
1812   case Bytecodes::_f2i:
1813   {
1814     Label L;
1815     __ cvttss2sil(rax, xmm0);
1816     __ cmpl(rax, 0x80000000); // NaN or overflow/underflow?
1817     __ jcc(Assembler::notEqual, L);
1818     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2i), 1);
1819     __ bind(L);
1820   }
1821     break;
1822   case Bytecodes::_f2l:
1823   {
1824     Label L;
1825     __ cvttss2siq(rax, xmm0);
1826     // NaN or overflow/underflow?
1827     __ cmp64(rax, ExternalAddress((address) &is_nan), rscratch1);
1828     __ jcc(Assembler::notEqual, L);
1829     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2l), 1);
1830     __ bind(L);
1831   }
1832     break;
1833   case Bytecodes::_f2d:
1834     __ cvtss2sd(xmm0, xmm0);
1835     break;
1836   case Bytecodes::_d2i:
1837   {
1838     Label L;
1839     __ cvttsd2sil(rax, xmm0);
1840     __ cmpl(rax, 0x80000000); // NaN or overflow/underflow?
1841     __ jcc(Assembler::notEqual, L);
1842     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2i), 1);
1843     __ bind(L);
1844   }
1845     break;
1846   case Bytecodes::_d2l:
1847   {
1848     Label L;
1849     __ cvttsd2siq(rax, xmm0);
1850     // NaN or overflow/underflow?
1851     __ cmp64(rax, ExternalAddress((address) &is_nan), rscratch1);
1852     __ jcc(Assembler::notEqual, L);
1853     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2l), 1);
1854     __ bind(L);
1855   }
1856     break;
1857   case Bytecodes::_d2f:
1858     __ cvtsd2ss(xmm0, xmm0);
1859     break;
1860   default:
1861     ShouldNotReachHere();
1862   }
1863 #else // !_LP64
1864   // Checking
1865 #ifdef ASSERT
1866   { TosState tos_in  = ilgl;
1867     TosState tos_out = ilgl;
1868     switch (bytecode()) {
1869       case Bytecodes::_i2l: // fall through
1870       case Bytecodes::_i2f: // fall through
1871       case Bytecodes::_i2d: // fall through
1872       case Bytecodes::_i2b: // fall through
1873       case Bytecodes::_i2c: // fall through
1874       case Bytecodes::_i2s: tos_in = itos; break;
1875       case Bytecodes::_l2i: // fall through
1876       case Bytecodes::_l2f: // fall through
1877       case Bytecodes::_l2d: tos_in = ltos; break;
1878       case Bytecodes::_f2i: // fall through
1879       case Bytecodes::_f2l: // fall through
1880       case Bytecodes::_f2d: tos_in = ftos; break;
1881       case Bytecodes::_d2i: // fall through
1882       case Bytecodes::_d2l: // fall through
1883       case Bytecodes::_d2f: tos_in = dtos; break;
1884       default             : ShouldNotReachHere();
1885     }
1886     switch (bytecode()) {
1887       case Bytecodes::_l2i: // fall through
1888       case Bytecodes::_f2i: // fall through
1889       case Bytecodes::_d2i: // fall through
1890       case Bytecodes::_i2b: // fall through
1891       case Bytecodes::_i2c: // fall through
1892       case Bytecodes::_i2s: tos_out = itos; break;
1893       case Bytecodes::_i2l: // fall through
1894       case Bytecodes::_f2l: // fall through
1895       case Bytecodes::_d2l: tos_out = ltos; break;
1896       case Bytecodes::_i2f: // fall through
1897       case Bytecodes::_l2f: // fall through
1898       case Bytecodes::_d2f: tos_out = ftos; break;
1899       case Bytecodes::_i2d: // fall through
1900       case Bytecodes::_l2d: // fall through
1901       case Bytecodes::_f2d: tos_out = dtos; break;
1902       default             : ShouldNotReachHere();
1903     }
1904     transition(tos_in, tos_out);
1905   }
1906 #endif // ASSERT
1907 
1908   // Conversion
1909   // (Note: use push(rcx)/pop(rcx) for 1/2-word stack-ptr manipulation)
1910   switch (bytecode()) {
1911     case Bytecodes::_i2l:
1912       __ extend_sign(rdx, rax);
1913       break;
1914     case Bytecodes::_i2f:
1915       if (UseSSE >= 1) {
1916         __ cvtsi2ssl(xmm0, rax);
1917       } else {
1918         __ push(rax);          // store int on tos
1919         __ fild_s(at_rsp());   // load int to ST0
1920         __ f2ieee();           // truncate to float size
1921         __ pop(rcx);           // adjust rsp
1922       }
1923       break;
1924     case Bytecodes::_i2d:
1925       if (UseSSE >= 2) {
1926         __ cvtsi2sdl(xmm0, rax);
1927       } else {
1928       __ push(rax);          // add one slot for d2ieee()
1929       __ push(rax);          // store int on tos
1930       __ fild_s(at_rsp());   // load int to ST0
1931       __ d2ieee();           // truncate to double size
1932       __ pop(rcx);           // adjust rsp
1933       __ pop(rcx);
1934       }
1935       break;
1936     case Bytecodes::_i2b:
1937       __ shll(rax, 24);      // truncate upper 24 bits
1938       __ sarl(rax, 24);      // and sign-extend byte
1939       LP64_ONLY(__ movsbl(rax, rax));
1940       break;
1941     case Bytecodes::_i2c:
1942       __ andl(rax, 0xFFFF);  // truncate upper 16 bits
1943       LP64_ONLY(__ movzwl(rax, rax));
1944       break;
1945     case Bytecodes::_i2s:
1946       __ shll(rax, 16);      // truncate upper 16 bits
1947       __ sarl(rax, 16);      // and sign-extend short
1948       LP64_ONLY(__ movswl(rax, rax));
1949       break;
1950     case Bytecodes::_l2i:
1951       /* nothing to do */
1952       break;
1953     case Bytecodes::_l2f:
1954       // On 64-bit platforms, the cvtsi2ssq instruction is used to convert
1955       // 64-bit long values to floats. On 32-bit platforms it is not possible
1956       // to use that instruction with 64-bit operands, therefore the FPU is
1957       // used to perform the conversion.
1958       __ push(rdx);          // store long on tos
1959       __ push(rax);
1960       __ fild_d(at_rsp());   // load long to ST0
1961       __ f2ieee();           // truncate to float size
1962       __ pop(rcx);           // adjust rsp
1963       __ pop(rcx);
1964       if (UseSSE >= 1) {
1965         __ push_f();
1966         __ pop_f(xmm0);
1967       }
1968       break;
1969     case Bytecodes::_l2d:
1970       // On 32-bit platforms the FPU is used for conversion because on
1971       // 32-bit platforms it is not not possible to use the cvtsi2sdq
1972       // instruction with 64-bit operands.
1973       __ push(rdx);          // store long on tos
1974       __ push(rax);
1975       __ fild_d(at_rsp());   // load long to ST0
1976       __ d2ieee();           // truncate to double size
1977       __ pop(rcx);           // adjust rsp
1978       __ pop(rcx);
1979       if (UseSSE >= 2) {
1980         __ push_d();
1981         __ pop_d(xmm0);
1982       }
1983       break;
1984     case Bytecodes::_f2i:
1985       // SharedRuntime::f2i does not differentiate between sNaNs and qNaNs
1986       // as it returns 0 for any NaN.
1987       if (UseSSE >= 1) {
1988         __ push_f(xmm0);
1989       } else {
1990         __ push(rcx);          // reserve space for argument
1991         __ fstp_s(at_rsp());   // pass float argument on stack
1992       }
1993       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2i), 1);
1994       break;
1995     case Bytecodes::_f2l:
1996       // SharedRuntime::f2l does not differentiate between sNaNs and qNaNs
1997       // as it returns 0 for any NaN.
1998       if (UseSSE >= 1) {
1999        __ push_f(xmm0);
2000       } else {
2001         __ push(rcx);          // reserve space for argument
2002         __ fstp_s(at_rsp());   // pass float argument on stack
2003       }
2004       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2l), 1);
2005       break;
2006     case Bytecodes::_f2d:
2007       if (UseSSE < 1) {
2008         /* nothing to do */
2009       } else if (UseSSE == 1) {
2010         __ push_f(xmm0);
2011         __ pop_f();
2012       } else { // UseSSE >= 2
2013         __ cvtss2sd(xmm0, xmm0);
2014       }
2015       break;
2016     case Bytecodes::_d2i:
2017       if (UseSSE >= 2) {
2018         __ push_d(xmm0);
2019       } else {
2020         __ push(rcx);          // reserve space for argument
2021         __ push(rcx);
2022         __ fstp_d(at_rsp());   // pass double argument on stack
2023       }
2024       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2i), 2);
2025       break;
2026     case Bytecodes::_d2l:
2027       if (UseSSE >= 2) {
2028         __ push_d(xmm0);
2029       } else {
2030         __ push(rcx);          // reserve space for argument
2031         __ push(rcx);
2032         __ fstp_d(at_rsp());   // pass double argument on stack
2033       }
2034       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2l), 2);
2035       break;
2036     case Bytecodes::_d2f:
2037       if (UseSSE <= 1) {
2038         __ push(rcx);          // reserve space for f2ieee()
2039         __ f2ieee();           // truncate to float size
2040         __ pop(rcx);           // adjust rsp
2041         if (UseSSE == 1) {
2042           // The cvtsd2ss instruction is not available if UseSSE==1, therefore
2043           // the conversion is performed using the FPU in this case.
2044           __ push_f();
2045           __ pop_f(xmm0);
2046         }
2047       } else { // UseSSE >= 2
2048         __ cvtsd2ss(xmm0, xmm0);
2049       }
2050       break;
2051     default             :
2052       ShouldNotReachHere();
2053   }
2054 #endif // _LP64
2055 }
2056 
2057 void TemplateTable::lcmp() {
2058   transition(ltos, itos);
2059 #ifdef _LP64
2060   Label done;
2061   __ pop_l(rdx);
2062   __ cmpq(rdx, rax);
2063   __ movl(rax, -1);
2064   __ jccb(Assembler::less, done);
2065   __ setb(Assembler::notEqual, rax);
2066   __ movzbl(rax, rax);
2067   __ bind(done);
2068 #else
2069 
2070   // y = rdx:rax
2071   __ pop_l(rbx, rcx);             // get x = rcx:rbx
2072   __ lcmp2int(rcx, rbx, rdx, rax);// rcx := cmp(x, y)
2073   __ mov(rax, rcx);
2074 #endif
2075 }
2076 
2077 void TemplateTable::float_cmp(bool is_float, int unordered_result) {
2078   if ((is_float && UseSSE >= 1) ||
2079       (!is_float && UseSSE >= 2)) {
2080     Label done;
2081     if (is_float) {
2082       // XXX get rid of pop here, use ... reg, mem32
2083       __ pop_f(xmm1);
2084       __ ucomiss(xmm1, xmm0);
2085     } else {
2086       // XXX get rid of pop here, use ... reg, mem64
2087       __ pop_d(xmm1);
2088       __ ucomisd(xmm1, xmm0);
2089     }
2090     if (unordered_result < 0) {
2091       __ movl(rax, -1);
2092       __ jccb(Assembler::parity, done);
2093       __ jccb(Assembler::below, done);
2094       __ setb(Assembler::notEqual, rdx);
2095       __ movzbl(rax, rdx);
2096     } else {
2097       __ movl(rax, 1);
2098       __ jccb(Assembler::parity, done);
2099       __ jccb(Assembler::above, done);
2100       __ movl(rax, 0);
2101       __ jccb(Assembler::equal, done);
2102       __ decrementl(rax);
2103     }
2104     __ bind(done);
2105   } else {
2106 #ifdef _LP64
2107     ShouldNotReachHere();
2108 #else // !_LP64
2109     if (is_float) {
2110       __ fld_s(at_rsp());
2111     } else {
2112       __ fld_d(at_rsp());
2113       __ pop(rdx);
2114     }
2115     __ pop(rcx);
2116     __ fcmp2int(rax, unordered_result < 0);
2117 #endif // _LP64
2118   }
2119 }
2120 
2121 void TemplateTable::branch(bool is_jsr, bool is_wide) {
2122   __ get_method(rcx); // rcx holds method
2123   __ profile_taken_branch(rax, rbx); // rax holds updated MDP, rbx
2124                                      // holds bumped taken count
2125 
2126   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
2127                              InvocationCounter::counter_offset();
2128   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
2129                               InvocationCounter::counter_offset();
2130 
2131   // Load up edx with the branch displacement
2132   if (is_wide) {
2133     __ movl(rdx, at_bcp(1));
2134   } else {
2135     __ load_signed_short(rdx, at_bcp(1));
2136   }
2137   __ bswapl(rdx);
2138 
2139   if (!is_wide) {
2140     __ sarl(rdx, 16);
2141   }
2142   LP64_ONLY(__ movl2ptr(rdx, rdx));
2143 
2144   // Handle all the JSR stuff here, then exit.
2145   // It's much shorter and cleaner than intermingling with the non-JSR
2146   // normal-branch stuff occurring below.
2147   if (is_jsr) {
2148     // Pre-load the next target bytecode into rbx
2149     __ load_unsigned_byte(rbx, Address(rbcp, rdx, Address::times_1, 0));
2150 
2151     // compute return address as bci in rax
2152     __ lea(rax, at_bcp((is_wide ? 5 : 3) -
2153                         in_bytes(ConstMethod::codes_offset())));
2154     __ subptr(rax, Address(rcx, Method::const_offset()));
2155     // Adjust the bcp in r13 by the displacement in rdx
2156     __ addptr(rbcp, rdx);
2157     // jsr returns atos that is not an oop
2158     __ push_i(rax);
2159     __ dispatch_only(vtos, true);
2160     return;
2161   }
2162 
2163   // Normal (non-jsr) branch handling
2164 
2165   // Adjust the bcp in r13 by the displacement in rdx
2166   __ addptr(rbcp, rdx);
2167 
2168   assert(UseLoopCounter || !UseOnStackReplacement,
2169          "on-stack-replacement requires loop counters");
2170   Label backedge_counter_overflow;
2171   Label dispatch;
2172   if (UseLoopCounter) {
2173     // increment backedge counter for backward branches
2174     // rax: MDO
2175     // rbx: MDO bumped taken-count
2176     // rcx: method
2177     // rdx: target offset
2178     // r13: target bcp
2179     // r14: locals pointer
2180     __ testl(rdx, rdx);             // check if forward or backward branch
2181     __ jcc(Assembler::positive, dispatch); // count only if backward branch
2182 
2183     // check if MethodCounters exists
2184     Label has_counters;
2185     __ movptr(rax, Address(rcx, Method::method_counters_offset()));
2186     __ testptr(rax, rax);
2187     __ jcc(Assembler::notZero, has_counters);
2188     __ push(rdx);
2189     __ push(rcx);
2190     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::build_method_counters),
2191                rcx);
2192     __ pop(rcx);
2193     __ pop(rdx);
2194     __ movptr(rax, Address(rcx, Method::method_counters_offset()));
2195     __ testptr(rax, rax);
2196     __ jcc(Assembler::zero, dispatch);
2197     __ bind(has_counters);
2198 
2199     Label no_mdo;
2200     if (ProfileInterpreter) {
2201       // Are we profiling?
2202       __ movptr(rbx, Address(rcx, in_bytes(Method::method_data_offset())));
2203       __ testptr(rbx, rbx);
2204       __ jccb(Assembler::zero, no_mdo);
2205       // Increment the MDO backedge counter
2206       const Address mdo_backedge_counter(rbx, in_bytes(MethodData::backedge_counter_offset()) +
2207           in_bytes(InvocationCounter::counter_offset()));
2208       const Address mask(rbx, in_bytes(MethodData::backedge_mask_offset()));
2209       __ increment_mask_and_jump(mdo_backedge_counter, mask, rax,
2210           UseOnStackReplacement ? &backedge_counter_overflow : nullptr);
2211       __ jmp(dispatch);
2212     }
2213     __ bind(no_mdo);
2214     // Increment backedge counter in MethodCounters*
2215     __ movptr(rcx, Address(rcx, Method::method_counters_offset()));
2216     const Address mask(rcx, in_bytes(MethodCounters::backedge_mask_offset()));
2217     __ increment_mask_and_jump(Address(rcx, be_offset), mask, rax,
2218         UseOnStackReplacement ? &backedge_counter_overflow : nullptr);
2219     __ bind(dispatch);
2220   }
2221 
2222   // Pre-load the next target bytecode into rbx
2223   __ load_unsigned_byte(rbx, Address(rbcp, 0));
2224 
2225   // continue with the bytecode @ target
2226   // rax: return bci for jsr's, unused otherwise
2227   // rbx: target bytecode
2228   // r13: target bcp
2229   __ dispatch_only(vtos, true);
2230 
2231   if (UseLoopCounter) {
2232     if (UseOnStackReplacement) {
2233       Label set_mdp;
2234       // invocation counter overflow
2235       __ bind(backedge_counter_overflow);
2236       __ negptr(rdx);
2237       __ addptr(rdx, rbcp); // branch bcp
2238       // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
2239       __ call_VM(noreg,
2240                  CAST_FROM_FN_PTR(address,
2241                                   InterpreterRuntime::frequency_counter_overflow),
2242                  rdx);
2243 
2244       // rax: osr nmethod (osr ok) or null (osr not possible)
2245       // rdx: scratch
2246       // r14: locals pointer
2247       // r13: bcp
2248       __ testptr(rax, rax);                        // test result
2249       __ jcc(Assembler::zero, dispatch);         // no osr if null
2250       // nmethod may have been invalidated (VM may block upon call_VM return)
2251       __ cmpb(Address(rax, nmethod::state_offset()), nmethod::in_use);
2252       __ jcc(Assembler::notEqual, dispatch);
2253 
2254       // We have the address of an on stack replacement routine in rax.
2255       // In preparation of invoking it, first we must migrate the locals
2256       // and monitors from off the interpreter frame on the stack.
2257       // Ensure to save the osr nmethod over the migration call,
2258       // it will be preserved in rbx.
2259       __ mov(rbx, rax);
2260 
2261       NOT_LP64(__ get_thread(rcx));
2262 
2263       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
2264 
2265       // rax is OSR buffer, move it to expected parameter location
2266       LP64_ONLY(__ mov(j_rarg0, rax));
2267       NOT_LP64(__ mov(rcx, rax));
2268       // We use j_rarg definitions here so that registers don't conflict as parameter
2269       // registers change across platforms as we are in the midst of a calling
2270       // sequence to the OSR nmethod and we don't want collision. These are NOT parameters.
2271 
2272       const Register retaddr   = LP64_ONLY(j_rarg2) NOT_LP64(rdi);
2273       const Register sender_sp = LP64_ONLY(j_rarg1) NOT_LP64(rdx);
2274 
2275       // pop the interpreter frame
2276       __ movptr(sender_sp, Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize)); // get sender sp
2277       __ leave();                                // remove frame anchor
2278       __ pop(retaddr);                           // get return address
2279       __ mov(rsp, sender_sp);                   // set sp to sender sp
2280       // Ensure compiled code always sees stack at proper alignment
2281       __ andptr(rsp, -(StackAlignmentInBytes));
2282 
2283       // unlike x86 we need no specialized return from compiled code
2284       // to the interpreter or the call stub.
2285 
2286       // push the return address
2287       __ push(retaddr);
2288 
2289       // and begin the OSR nmethod
2290       __ jmp(Address(rbx, nmethod::osr_entry_point_offset()));
2291     }
2292   }
2293 }
2294 
2295 void TemplateTable::if_0cmp(Condition cc) {
2296   transition(itos, vtos);
2297   // assume branch is more often taken than not (loops use backward branches)
2298   Label not_taken;
2299   __ testl(rax, rax);
2300   __ jcc(j_not(cc), not_taken);
2301   branch(false, false);
2302   __ bind(not_taken);
2303   __ profile_not_taken_branch(rax);
2304 }
2305 
2306 void TemplateTable::if_icmp(Condition cc) {
2307   transition(itos, vtos);
2308   // assume branch is more often taken than not (loops use backward branches)
2309   Label not_taken;
2310   __ pop_i(rdx);
2311   __ cmpl(rdx, rax);
2312   __ jcc(j_not(cc), not_taken);
2313   branch(false, false);
2314   __ bind(not_taken);
2315   __ profile_not_taken_branch(rax);
2316 }
2317 
2318 void TemplateTable::if_nullcmp(Condition cc) {
2319   transition(atos, vtos);
2320   // assume branch is more often taken than not (loops use backward branches)
2321   Label not_taken;
2322   __ testptr(rax, rax);
2323   __ jcc(j_not(cc), not_taken);
2324   branch(false, false);
2325   __ bind(not_taken);
2326   __ profile_not_taken_branch(rax);
2327 }
2328 
2329 void TemplateTable::if_acmp(Condition cc) {
2330   transition(atos, vtos);
2331   // assume branch is more often taken than not (loops use backward branches)
2332   Label not_taken;
2333   __ pop_ptr(rdx);
2334   __ cmpoop(rdx, rax);
2335   __ jcc(j_not(cc), not_taken);
2336   branch(false, false);
2337   __ bind(not_taken);
2338   __ profile_not_taken_branch(rax);
2339 }
2340 
2341 void TemplateTable::ret() {
2342   transition(vtos, vtos);
2343   locals_index(rbx);
2344   LP64_ONLY(__ movslq(rbx, iaddress(rbx))); // get return bci, compute return bcp
2345   NOT_LP64(__ movptr(rbx, iaddress(rbx)));
2346   __ profile_ret(rbx, rcx);
2347   __ get_method(rax);
2348   __ movptr(rbcp, Address(rax, Method::const_offset()));
2349   __ lea(rbcp, Address(rbcp, rbx, Address::times_1,
2350                       ConstMethod::codes_offset()));
2351   __ dispatch_next(vtos, 0, true);
2352 }
2353 
2354 void TemplateTable::wide_ret() {
2355   transition(vtos, vtos);
2356   locals_index_wide(rbx);
2357   __ movptr(rbx, aaddress(rbx)); // get return bci, compute return bcp
2358   __ profile_ret(rbx, rcx);
2359   __ get_method(rax);
2360   __ movptr(rbcp, Address(rax, Method::const_offset()));
2361   __ lea(rbcp, Address(rbcp, rbx, Address::times_1, ConstMethod::codes_offset()));
2362   __ dispatch_next(vtos, 0, true);
2363 }
2364 
2365 void TemplateTable::tableswitch() {
2366   Label default_case, continue_execution;
2367   transition(itos, vtos);
2368 
2369   // align r13/rsi
2370   __ lea(rbx, at_bcp(BytesPerInt));
2371   __ andptr(rbx, -BytesPerInt);
2372   // load lo & hi
2373   __ movl(rcx, Address(rbx, BytesPerInt));
2374   __ movl(rdx, Address(rbx, 2 * BytesPerInt));
2375   __ bswapl(rcx);
2376   __ bswapl(rdx);
2377   // check against lo & hi
2378   __ cmpl(rax, rcx);
2379   __ jcc(Assembler::less, default_case);
2380   __ cmpl(rax, rdx);
2381   __ jcc(Assembler::greater, default_case);
2382   // lookup dispatch offset
2383   __ subl(rax, rcx);
2384   __ movl(rdx, Address(rbx, rax, Address::times_4, 3 * BytesPerInt));
2385   __ profile_switch_case(rax, rbx, rcx);
2386   // continue execution
2387   __ bind(continue_execution);
2388   __ bswapl(rdx);
2389   LP64_ONLY(__ movl2ptr(rdx, rdx));
2390   __ load_unsigned_byte(rbx, Address(rbcp, rdx, Address::times_1));
2391   __ addptr(rbcp, rdx);
2392   __ dispatch_only(vtos, true);
2393   // handle default
2394   __ bind(default_case);
2395   __ profile_switch_default(rax);
2396   __ movl(rdx, Address(rbx, 0));
2397   __ jmp(continue_execution);
2398 }
2399 
2400 void TemplateTable::lookupswitch() {
2401   transition(itos, itos);
2402   __ stop("lookupswitch bytecode should have been rewritten");
2403 }
2404 
2405 void TemplateTable::fast_linearswitch() {
2406   transition(itos, vtos);
2407   Label loop_entry, loop, found, continue_execution;
2408   // bswap rax so we can avoid bswapping the table entries
2409   __ bswapl(rax);
2410   // align r13
2411   __ lea(rbx, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2412                                     // this instruction (change offsets
2413                                     // below)
2414   __ andptr(rbx, -BytesPerInt);
2415   // set counter
2416   __ movl(rcx, Address(rbx, BytesPerInt));
2417   __ bswapl(rcx);
2418   __ jmpb(loop_entry);
2419   // table search
2420   __ bind(loop);
2421   __ cmpl(rax, Address(rbx, rcx, Address::times_8, 2 * BytesPerInt));
2422   __ jcc(Assembler::equal, found);
2423   __ bind(loop_entry);
2424   __ decrementl(rcx);
2425   __ jcc(Assembler::greaterEqual, loop);
2426   // default case
2427   __ profile_switch_default(rax);
2428   __ movl(rdx, Address(rbx, 0));
2429   __ jmp(continue_execution);
2430   // entry found -> get offset
2431   __ bind(found);
2432   __ movl(rdx, Address(rbx, rcx, Address::times_8, 3 * BytesPerInt));
2433   __ profile_switch_case(rcx, rax, rbx);
2434   // continue execution
2435   __ bind(continue_execution);
2436   __ bswapl(rdx);
2437   __ movl2ptr(rdx, rdx);
2438   __ load_unsigned_byte(rbx, Address(rbcp, rdx, Address::times_1));
2439   __ addptr(rbcp, rdx);
2440   __ dispatch_only(vtos, true);
2441 }
2442 
2443 void TemplateTable::fast_binaryswitch() {
2444   transition(itos, vtos);
2445   // Implementation using the following core algorithm:
2446   //
2447   // int binary_search(int key, LookupswitchPair* array, int n) {
2448   //   // Binary search according to "Methodik des Programmierens" by
2449   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2450   //   int i = 0;
2451   //   int j = n;
2452   //   while (i+1 < j) {
2453   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2454   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2455   //     // where a stands for the array and assuming that the (inexisting)
2456   //     // element a[n] is infinitely big.
2457   //     int h = (i + j) >> 1;
2458   //     // i < h < j
2459   //     if (key < array[h].fast_match()) {
2460   //       j = h;
2461   //     } else {
2462   //       i = h;
2463   //     }
2464   //   }
2465   //   // R: a[i] <= key < a[i+1] or Q
2466   //   // (i.e., if key is within array, i is the correct index)
2467   //   return i;
2468   // }
2469 
2470   // Register allocation
2471   const Register key   = rax; // already set (tosca)
2472   const Register array = rbx;
2473   const Register i     = rcx;
2474   const Register j     = rdx;
2475   const Register h     = rdi;
2476   const Register temp  = rsi;
2477 
2478   // Find array start
2479   NOT_LP64(__ save_bcp());
2480 
2481   __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2482                                           // get rid of this
2483                                           // instruction (change
2484                                           // offsets below)
2485   __ andptr(array, -BytesPerInt);
2486 
2487   // Initialize i & j
2488   __ xorl(i, i);                            // i = 0;
2489   __ movl(j, Address(array, -BytesPerInt)); // j = length(array);
2490 
2491   // Convert j into native byteordering
2492   __ bswapl(j);
2493 
2494   // And start
2495   Label entry;
2496   __ jmp(entry);
2497 
2498   // binary search loop
2499   {
2500     Label loop;
2501     __ bind(loop);
2502     // int h = (i + j) >> 1;
2503     __ leal(h, Address(i, j, Address::times_1)); // h = i + j;
2504     __ sarl(h, 1);                               // h = (i + j) >> 1;
2505     // if (key < array[h].fast_match()) {
2506     //   j = h;
2507     // } else {
2508     //   i = h;
2509     // }
2510     // Convert array[h].match to native byte-ordering before compare
2511     __ movl(temp, Address(array, h, Address::times_8));
2512     __ bswapl(temp);
2513     __ cmpl(key, temp);
2514     // j = h if (key <  array[h].fast_match())
2515     __ cmov32(Assembler::less, j, h);
2516     // i = h if (key >= array[h].fast_match())
2517     __ cmov32(Assembler::greaterEqual, i, h);
2518     // while (i+1 < j)
2519     __ bind(entry);
2520     __ leal(h, Address(i, 1)); // i+1
2521     __ cmpl(h, j);             // i+1 < j
2522     __ jcc(Assembler::less, loop);
2523   }
2524 
2525   // end of binary search, result index is i (must check again!)
2526   Label default_case;
2527   // Convert array[i].match to native byte-ordering before compare
2528   __ movl(temp, Address(array, i, Address::times_8));
2529   __ bswapl(temp);
2530   __ cmpl(key, temp);
2531   __ jcc(Assembler::notEqual, default_case);
2532 
2533   // entry found -> j = offset
2534   __ movl(j , Address(array, i, Address::times_8, BytesPerInt));
2535   __ profile_switch_case(i, key, array);
2536   __ bswapl(j);
2537   LP64_ONLY(__ movslq(j, j));
2538 
2539   NOT_LP64(__ restore_bcp());
2540   NOT_LP64(__ restore_locals());                           // restore rdi
2541 
2542   __ load_unsigned_byte(rbx, Address(rbcp, j, Address::times_1));
2543   __ addptr(rbcp, j);
2544   __ dispatch_only(vtos, true);
2545 
2546   // default case -> j = default offset
2547   __ bind(default_case);
2548   __ profile_switch_default(i);
2549   __ movl(j, Address(array, -2 * BytesPerInt));
2550   __ bswapl(j);
2551   LP64_ONLY(__ movslq(j, j));
2552 
2553   NOT_LP64(__ restore_bcp());
2554   NOT_LP64(__ restore_locals());
2555 
2556   __ load_unsigned_byte(rbx, Address(rbcp, j, Address::times_1));
2557   __ addptr(rbcp, j);
2558   __ dispatch_only(vtos, true);
2559 }
2560 
2561 void TemplateTable::_return(TosState state) {
2562   transition(state, state);
2563 
2564   assert(_desc->calls_vm(),
2565          "inconsistent calls_vm information"); // call in remove_activation
2566 
2567   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2568     assert(state == vtos, "only valid state");
2569     Register robj = LP64_ONLY(c_rarg1) NOT_LP64(rax);
2570     __ movptr(robj, aaddress(0));
2571     __ load_klass(rdi, robj, rscratch1);
2572     __ movl(rdi, Address(rdi, Klass::access_flags_offset()));
2573     __ testl(rdi, JVM_ACC_HAS_FINALIZER);
2574     Label skip_register_finalizer;
2575     __ jcc(Assembler::zero, skip_register_finalizer);
2576 
2577     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), robj);
2578 
2579     __ bind(skip_register_finalizer);
2580   }
2581 
2582   if (_desc->bytecode() != Bytecodes::_return_register_finalizer) {
2583     Label no_safepoint;
2584     NOT_PRODUCT(__ block_comment("Thread-local Safepoint poll"));
2585 #ifdef _LP64
2586     __ testb(Address(r15_thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
2587 #else
2588     const Register thread = rdi;
2589     __ get_thread(thread);
2590     __ testb(Address(thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
2591 #endif
2592     __ jcc(Assembler::zero, no_safepoint);
2593     __ push(state);
2594     __ push_cont_fastpath();
2595     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2596                                        InterpreterRuntime::at_safepoint));
2597     __ pop_cont_fastpath();
2598     __ pop(state);
2599     __ bind(no_safepoint);
2600   }
2601 
2602   // Narrow result if state is itos but result type is smaller.
2603   // Need to narrow in the return bytecode rather than in generate_return_entry
2604   // since compiled code callers expect the result to already be narrowed.
2605   if (state == itos) {
2606     __ narrow(rax);
2607   }
2608   __ remove_activation(state, rbcp);
2609 
2610   __ jmp(rbcp);
2611 }
2612 
2613 // ----------------------------------------------------------------------------
2614 // Volatile variables demand their effects be made known to all CPU's
2615 // in order.  Store buffers on most chips allow reads & writes to
2616 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2617 // without some kind of memory barrier (i.e., it's not sufficient that
2618 // the interpreter does not reorder volatile references, the hardware
2619 // also must not reorder them).
2620 //
2621 // According to the new Java Memory Model (JMM):
2622 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2623 //     writes act as acquire & release, so:
2624 // (2) A read cannot let unrelated NON-volatile memory refs that
2625 //     happen after the read float up to before the read.  It's OK for
2626 //     non-volatile memory refs that happen before the volatile read to
2627 //     float down below it.
2628 // (3) Similar a volatile write cannot let unrelated NON-volatile
2629 //     memory refs that happen BEFORE the write float down to after the
2630 //     write.  It's OK for non-volatile memory refs that happen after the
2631 //     volatile write to float up before it.
2632 //
2633 // We only put in barriers around volatile refs (they are expensive),
2634 // not _between_ memory refs (that would require us to track the
2635 // flavor of the previous memory refs).  Requirements (2) and (3)
2636 // require some barriers before volatile stores and after volatile
2637 // loads.  These nearly cover requirement (1) but miss the
2638 // volatile-store-volatile-load case.  This final case is placed after
2639 // volatile-stores although it could just as well go before
2640 // volatile-loads.
2641 
2642 void TemplateTable::volatile_barrier(Assembler::Membar_mask_bits order_constraint ) {
2643   // Helper function to insert a is-volatile test and memory barrier
2644   __ membar(order_constraint);
2645 }
2646 
2647 void TemplateTable::resolve_cache_and_index(int byte_no,
2648                                             Register cache,
2649                                             Register index,
2650                                             size_t index_size) {
2651   const Register temp = rbx;
2652   assert_different_registers(cache, index, temp);
2653 
2654   Label L_clinit_barrier_slow;
2655   Label resolved;
2656 
2657   Bytecodes::Code code = bytecode();
2658   switch (code) {
2659   case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2660   case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2661   default: break;
2662   }
2663 
2664   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2665   __ get_cache_and_index_and_bytecode_at_bcp(cache, index, temp, byte_no, 1, index_size);
2666   __ cmpl(temp, code);  // have we resolved this bytecode?
2667   __ jcc(Assembler::equal, resolved);
2668 
2669   // resolve first time through
2670   // Class initialization barrier slow path lands here as well.
2671   __ bind(L_clinit_barrier_slow);
2672   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2673   __ movl(temp, code);
2674   __ call_VM(noreg, entry, temp);
2675   // Update registers with resolved info
2676   __ get_cache_and_index_at_bcp(cache, index, 1, index_size);
2677 
2678   __ bind(resolved);
2679 
2680   // Class initialization barrier for static methods
2681   if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
2682     const Register method = temp;
2683     const Register klass  = temp;
2684     const Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2685     assert(thread != noreg, "x86_32 not supported");
2686 
2687     __ load_resolved_method_at_index(byte_no, method, cache, index);
2688     __ load_method_holder(klass, method);
2689     __ clinit_barrier(klass, thread, nullptr /*L_fast_path*/, &L_clinit_barrier_slow);
2690   }
2691 }
2692 
2693 // The cache and index registers must be set before call
2694 void TemplateTable::load_field_cp_cache_entry(Register obj,
2695                                               Register cache,
2696                                               Register index,
2697                                               Register off,
2698                                               Register flags,
2699                                               bool is_static = false) {
2700   assert_different_registers(cache, index, flags, off);
2701 
2702   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2703   // Field offset
2704   __ movptr(off, Address(cache, index, Address::times_ptr,
2705                          in_bytes(cp_base_offset +
2706                                   ConstantPoolCacheEntry::f2_offset())));
2707   // Flags
2708   __ movl(flags, Address(cache, index, Address::times_ptr,
2709                          in_bytes(cp_base_offset +
2710                                   ConstantPoolCacheEntry::flags_offset())));
2711 
2712   // klass overwrite register
2713   if (is_static) {
2714     __ movptr(obj, Address(cache, index, Address::times_ptr,
2715                            in_bytes(cp_base_offset +
2716                                     ConstantPoolCacheEntry::f1_offset())));
2717     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2718     __ movptr(obj, Address(obj, mirror_offset));
2719     __ resolve_oop_handle(obj, rscratch2);
2720   }
2721 }
2722 
2723 void TemplateTable::load_invokedynamic_entry(Register method) {
2724   // setup registers
2725   const Register appendix = rax;
2726   const Register cache = rcx;
2727   const Register index = rdx;
2728   assert_different_registers(method, appendix, cache, index);
2729 
2730   __ save_bcp();
2731 
2732   Label resolved;
2733 
2734   __ load_resolved_indy_entry(cache, index);
2735   __ movptr(method, Address(cache, in_bytes(ResolvedIndyEntry::method_offset())));
2736 
2737   // Compare the method to zero
2738   __ testptr(method, method);
2739   __ jcc(Assembler::notZero, resolved);
2740 
2741   Bytecodes::Code code = bytecode();
2742 
2743   // Call to the interpreter runtime to resolve invokedynamic
2744   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2745   __ movl(method, code); // this is essentially Bytecodes::_invokedynamic
2746   __ call_VM(noreg, entry, method);
2747   // Update registers with resolved info
2748   __ load_resolved_indy_entry(cache, index);
2749   __ movptr(method, Address(cache, in_bytes(ResolvedIndyEntry::method_offset())));
2750 
2751 #ifdef ASSERT
2752   __ testptr(method, method);
2753   __ jcc(Assembler::notZero, resolved);
2754   __ stop("Should be resolved by now");
2755 #endif // ASSERT
2756   __ bind(resolved);
2757 
2758   Label L_no_push;
2759   // Check if there is an appendix
2760   __ load_unsigned_byte(index, Address(cache, in_bytes(ResolvedIndyEntry::flags_offset())));
2761   __ testl(index, (1 << ResolvedIndyEntry::has_appendix_shift));
2762   __ jcc(Assembler::zero, L_no_push);
2763 
2764   // Get appendix
2765   __ load_unsigned_short(index, Address(cache, in_bytes(ResolvedIndyEntry::resolved_references_index_offset())));
2766   // Push the appendix as a trailing parameter
2767   // since the parameter_size includes it.
2768   __ load_resolved_reference_at_index(appendix, index);
2769   __ verify_oop(appendix);
2770   __ push(appendix);  // push appendix (MethodType, CallSite, etc.)
2771   __ bind(L_no_push);
2772 
2773   // compute return type
2774   __ load_unsigned_byte(index, Address(cache, in_bytes(ResolvedIndyEntry::result_type_offset())));
2775   // load return address
2776   {
2777     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
2778     ExternalAddress table(table_addr);
2779 #ifdef _LP64
2780     __ lea(rscratch1, table);
2781     __ movptr(index, Address(rscratch1, index, Address::times_ptr));
2782 #else
2783     __ movptr(index, ArrayAddress(table, Address(noreg, index, Address::times_ptr)));
2784 #endif // _LP64
2785   }
2786 
2787   // push return address
2788   __ push(index);
2789 }
2790 
2791 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2792                                                Register method,
2793                                                Register itable_index,
2794                                                Register flags,
2795                                                bool is_invokevirtual,
2796                                                bool is_invokevfinal, /*unused*/
2797                                                bool is_invokedynamic /*unused*/) {
2798   // setup registers
2799   const Register cache = rcx;
2800   const Register index = rdx;
2801   assert_different_registers(method, flags);
2802   assert_different_registers(method, cache, index);
2803   assert_different_registers(itable_index, flags);
2804   assert_different_registers(itable_index, cache, index);
2805   // determine constant pool cache field offsets
2806   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2807   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2808                                     ConstantPoolCacheEntry::flags_offset());
2809   // access constant pool cache fields
2810   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2811                                     ConstantPoolCacheEntry::f2_offset());
2812 
2813   size_t index_size = sizeof(u2);
2814   resolve_cache_and_index(byte_no, cache, index, index_size);
2815   __ load_resolved_method_at_index(byte_no, method, cache, index);
2816 
2817   if (itable_index != noreg) {
2818     // pick up itable or appendix index from f2 also:
2819     __ movptr(itable_index, Address(cache, index, Address::times_ptr, index_offset));
2820   }
2821   __ movl(flags, Address(cache, index, Address::times_ptr, flags_offset));
2822 }
2823 
2824 // The registers cache and index expected to be set before call.
2825 // Correct values of the cache and index registers are preserved.
2826 void TemplateTable::jvmti_post_field_access(Register cache,
2827                                             Register index,
2828                                             bool is_static,
2829                                             bool has_tos) {
2830   if (JvmtiExport::can_post_field_access()) {
2831     // Check to see if a field access watch has been set before we take
2832     // the time to call into the VM.
2833     Label L1;
2834     assert_different_registers(cache, index, rax);
2835     __ mov32(rax, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2836     __ testl(rax,rax);
2837     __ jcc(Assembler::zero, L1);
2838 
2839     // cache entry pointer
2840     __ addptr(cache, in_bytes(ConstantPoolCache::base_offset()));
2841     __ shll(index, LogBytesPerWord);
2842     __ addptr(cache, index);
2843     if (is_static) {
2844       __ xorptr(rax, rax);      // null object reference
2845     } else {
2846       __ pop(atos);         // Get the object
2847       __ verify_oop(rax);
2848       __ push(atos);        // Restore stack state
2849     }
2850     // rax,:   object pointer or null
2851     // cache: cache entry pointer
2852     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access),
2853                rax, cache);
2854     __ get_cache_and_index_at_bcp(cache, index, 1);
2855     __ bind(L1);
2856   }
2857 }
2858 
2859 void TemplateTable::pop_and_check_object(Register r) {
2860   __ pop_ptr(r);
2861   __ null_check(r);  // for field access must check obj.
2862   __ verify_oop(r);
2863 }
2864 
2865 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2866   transition(vtos, vtos);
2867 
2868   const Register cache = rcx;
2869   const Register index = rdx;
2870   const Register obj   = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
2871   const Register off   = rbx;
2872   const Register flags = rax;
2873   const Register bc    = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // uses same reg as obj, so don't mix them
2874 
2875   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2876   jvmti_post_field_access(cache, index, is_static, false);
2877   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2878 
2879   if (!is_static) pop_and_check_object(obj);
2880 
2881   const Address field(obj, off, Address::times_1, 0*wordSize);
2882 
2883   Label Done, notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj;
2884 
2885   __ shrl(flags, ConstantPoolCacheEntry::tos_state_shift);
2886   // Make sure we don't need to mask edx after the above shift
2887   assert(btos == 0, "change code, btos != 0");
2888 
2889   __ andl(flags, ConstantPoolCacheEntry::tos_state_mask);
2890 
2891   __ jcc(Assembler::notZero, notByte);
2892   // btos
2893   __ access_load_at(T_BYTE, IN_HEAP, rax, field, noreg, noreg);
2894   __ push(btos);
2895   // Rewrite bytecode to be faster
2896   if (!is_static && rc == may_rewrite) {
2897     patch_bytecode(Bytecodes::_fast_bgetfield, bc, rbx);
2898   }
2899   __ jmp(Done);
2900 
2901   __ bind(notByte);
2902   __ cmpl(flags, ztos);
2903   __ jcc(Assembler::notEqual, notBool);
2904 
2905   // ztos (same code as btos)
2906   __ access_load_at(T_BOOLEAN, IN_HEAP, rax, field, noreg, noreg);
2907   __ push(ztos);
2908   // Rewrite bytecode to be faster
2909   if (!is_static && rc == may_rewrite) {
2910     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2911     patch_bytecode(Bytecodes::_fast_bgetfield, bc, rbx);
2912   }
2913   __ jmp(Done);
2914 
2915   __ bind(notBool);
2916   __ cmpl(flags, atos);
2917   __ jcc(Assembler::notEqual, notObj);
2918   // atos
2919   do_oop_load(_masm, field, rax);
2920   __ push(atos);
2921   if (!is_static && rc == may_rewrite) {
2922     patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
2923   }
2924   __ jmp(Done);
2925 
2926   __ bind(notObj);
2927   __ cmpl(flags, itos);
2928   __ jcc(Assembler::notEqual, notInt);
2929   // itos
2930   __ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
2931   __ push(itos);
2932   // Rewrite bytecode to be faster
2933   if (!is_static && rc == may_rewrite) {
2934     patch_bytecode(Bytecodes::_fast_igetfield, bc, rbx);
2935   }
2936   __ jmp(Done);
2937 
2938   __ bind(notInt);
2939   __ cmpl(flags, ctos);
2940   __ jcc(Assembler::notEqual, notChar);
2941   // ctos
2942   __ access_load_at(T_CHAR, IN_HEAP, rax, field, noreg, noreg);
2943   __ push(ctos);
2944   // Rewrite bytecode to be faster
2945   if (!is_static && rc == may_rewrite) {
2946     patch_bytecode(Bytecodes::_fast_cgetfield, bc, rbx);
2947   }
2948   __ jmp(Done);
2949 
2950   __ bind(notChar);
2951   __ cmpl(flags, stos);
2952   __ jcc(Assembler::notEqual, notShort);
2953   // stos
2954   __ access_load_at(T_SHORT, IN_HEAP, rax, field, noreg, noreg);
2955   __ push(stos);
2956   // Rewrite bytecode to be faster
2957   if (!is_static && rc == may_rewrite) {
2958     patch_bytecode(Bytecodes::_fast_sgetfield, bc, rbx);
2959   }
2960   __ jmp(Done);
2961 
2962   __ bind(notShort);
2963   __ cmpl(flags, ltos);
2964   __ jcc(Assembler::notEqual, notLong);
2965   // ltos
2966     // Generate code as if volatile (x86_32).  There just aren't enough registers to
2967     // save that information and this code is faster than the test.
2968   __ access_load_at(T_LONG, IN_HEAP | MO_RELAXED, noreg /* ltos */, field, noreg, noreg);
2969   __ push(ltos);
2970   // Rewrite bytecode to be faster
2971   LP64_ONLY(if (!is_static && rc == may_rewrite) patch_bytecode(Bytecodes::_fast_lgetfield, bc, rbx));
2972   __ jmp(Done);
2973 
2974   __ bind(notLong);
2975   __ cmpl(flags, ftos);
2976   __ jcc(Assembler::notEqual, notFloat);
2977   // ftos
2978 
2979   __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
2980   __ push(ftos);
2981   // Rewrite bytecode to be faster
2982   if (!is_static && rc == may_rewrite) {
2983     patch_bytecode(Bytecodes::_fast_fgetfield, bc, rbx);
2984   }
2985   __ jmp(Done);
2986 
2987   __ bind(notFloat);
2988 #ifdef ASSERT
2989   Label notDouble;
2990   __ cmpl(flags, dtos);
2991   __ jcc(Assembler::notEqual, notDouble);
2992 #endif
2993   // dtos
2994   // MO_RELAXED: for the case of volatile field, in fact it adds no extra work for the underlying implementation
2995   __ access_load_at(T_DOUBLE, IN_HEAP | MO_RELAXED, noreg /* dtos */, field, noreg, noreg);
2996   __ push(dtos);
2997   // Rewrite bytecode to be faster
2998   if (!is_static && rc == may_rewrite) {
2999     patch_bytecode(Bytecodes::_fast_dgetfield, bc, rbx);
3000   }
3001 #ifdef ASSERT
3002   __ jmp(Done);
3003 
3004   __ bind(notDouble);
3005   __ stop("Bad state");
3006 #endif
3007 
3008   __ bind(Done);
3009   // [jk] not needed currently
3010   // volatile_barrier(Assembler::Membar_mask_bits(Assembler::LoadLoad |
3011   //                                              Assembler::LoadStore));
3012 }
3013 
3014 void TemplateTable::getfield(int byte_no) {
3015   getfield_or_static(byte_no, false);
3016 }
3017 
3018 void TemplateTable::nofast_getfield(int byte_no) {
3019   getfield_or_static(byte_no, false, may_not_rewrite);
3020 }
3021 
3022 void TemplateTable::getstatic(int byte_no) {
3023   getfield_or_static(byte_no, true);
3024 }
3025 
3026 
3027 // The registers cache and index expected to be set before call.
3028 // The function may destroy various registers, just not the cache and index registers.
3029 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
3030 
3031   const Register robj = LP64_ONLY(c_rarg2)   NOT_LP64(rax);
3032   const Register RBX  = LP64_ONLY(c_rarg1)   NOT_LP64(rbx);
3033   const Register RCX  = LP64_ONLY(c_rarg3)   NOT_LP64(rcx);
3034   const Register RDX  = LP64_ONLY(rscratch1) NOT_LP64(rdx);
3035 
3036   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
3037 
3038   if (JvmtiExport::can_post_field_modification()) {
3039     // Check to see if a field modification watch has been set before
3040     // we take the time to call into the VM.
3041     Label L1;
3042     assert_different_registers(cache, index, rax);
3043     __ mov32(rax, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
3044     __ testl(rax, rax);
3045     __ jcc(Assembler::zero, L1);
3046 
3047     __ get_cache_and_index_at_bcp(robj, RDX, 1);
3048 
3049 
3050     if (is_static) {
3051       // Life is simple.  Null out the object pointer.
3052       __ xorl(RBX, RBX);
3053 
3054     } else {
3055       // Life is harder. The stack holds the value on top, followed by
3056       // the object.  We don't know the size of the value, though; it
3057       // could be one or two words depending on its type. As a result,
3058       // we must find the type to determine where the object is.
3059 #ifndef _LP64
3060       Label two_word, valsize_known;
3061 #endif
3062       __ movl(RCX, Address(robj, RDX,
3063                            Address::times_ptr,
3064                            in_bytes(cp_base_offset +
3065                                      ConstantPoolCacheEntry::flags_offset())));
3066       NOT_LP64(__ mov(rbx, rsp));
3067       __ shrl(RCX, ConstantPoolCacheEntry::tos_state_shift);
3068 
3069       // Make sure we don't need to mask rcx after the above shift
3070       ConstantPoolCacheEntry::verify_tos_state_shift();
3071 #ifdef _LP64
3072       __ movptr(c_rarg1, at_tos_p1());  // initially assume a one word jvalue
3073       __ cmpl(c_rarg3, ltos);
3074       __ cmovptr(Assembler::equal,
3075                  c_rarg1, at_tos_p2()); // ltos (two word jvalue)
3076       __ cmpl(c_rarg3, dtos);
3077       __ cmovptr(Assembler::equal,
3078                  c_rarg1, at_tos_p2()); // dtos (two word jvalue)
3079 #else
3080       __ cmpl(rcx, ltos);
3081       __ jccb(Assembler::equal, two_word);
3082       __ cmpl(rcx, dtos);
3083       __ jccb(Assembler::equal, two_word);
3084       __ addptr(rbx, Interpreter::expr_offset_in_bytes(1)); // one word jvalue (not ltos, dtos)
3085       __ jmpb(valsize_known);
3086 
3087       __ bind(two_word);
3088       __ addptr(rbx, Interpreter::expr_offset_in_bytes(2)); // two words jvalue
3089 
3090       __ bind(valsize_known);
3091       // setup object pointer
3092       __ movptr(rbx, Address(rbx, 0));
3093 #endif
3094     }
3095     // cache entry pointer
3096     __ addptr(robj, in_bytes(cp_base_offset));
3097     __ shll(RDX, LogBytesPerWord);
3098     __ addptr(robj, RDX);
3099     // object (tos)
3100     __ mov(RCX, rsp);
3101     // c_rarg1: object pointer set up above (null if static)
3102     // c_rarg2: cache entry pointer
3103     // c_rarg3: jvalue object on the stack
3104     __ call_VM(noreg,
3105                CAST_FROM_FN_PTR(address,
3106                                 InterpreterRuntime::post_field_modification),
3107                RBX, robj, RCX);
3108     __ get_cache_and_index_at_bcp(cache, index, 1);
3109     __ bind(L1);
3110   }
3111 }
3112 
3113 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
3114   transition(vtos, vtos);
3115 
3116   const Register cache = rcx;
3117   const Register index = rdx;
3118   const Register obj   = rcx;
3119   const Register off   = rbx;
3120   const Register flags = rax;
3121 
3122   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
3123   jvmti_post_field_mod(cache, index, is_static);
3124   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
3125 
3126   // [jk] not needed currently
3127   // volatile_barrier(Assembler::Membar_mask_bits(Assembler::LoadStore |
3128   //                                              Assembler::StoreStore));
3129 
3130   Label notVolatile, Done;
3131   __ movl(rdx, flags);
3132   __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift);
3133   __ andl(rdx, 0x1);
3134 
3135   // Check for volatile store
3136   __ testl(rdx, rdx);
3137   __ jcc(Assembler::zero, notVolatile);
3138 
3139   putfield_or_static_helper(byte_no, is_static, rc, obj, off, flags);
3140   volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
3141                                                Assembler::StoreStore));
3142   __ jmp(Done);
3143   __ bind(notVolatile);
3144 
3145   putfield_or_static_helper(byte_no, is_static, rc, obj, off, flags);
3146 
3147   __ bind(Done);
3148 }
3149 
3150 void TemplateTable::putfield_or_static_helper(int byte_no, bool is_static, RewriteControl rc,
3151                                               Register obj, Register off, Register flags) {
3152 
3153   // field addresses
3154   const Address field(obj, off, Address::times_1, 0*wordSize);
3155   NOT_LP64( const Address hi(obj, off, Address::times_1, 1*wordSize);)
3156 
3157   Label notByte, notBool, notInt, notShort, notChar,
3158         notLong, notFloat, notObj;
3159   Label Done;
3160 
3161   const Register bc    = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
3162 
3163   __ shrl(flags, ConstantPoolCacheEntry::tos_state_shift);
3164 
3165   assert(btos == 0, "change code, btos != 0");
3166   __ andl(flags, ConstantPoolCacheEntry::tos_state_mask);
3167   __ jcc(Assembler::notZero, notByte);
3168 
3169   // btos
3170   {
3171     __ pop(btos);
3172     if (!is_static) pop_and_check_object(obj);
3173     __ access_store_at(T_BYTE, IN_HEAP, field, rax, noreg, noreg, noreg);
3174     if (!is_static && rc == may_rewrite) {
3175       patch_bytecode(Bytecodes::_fast_bputfield, bc, rbx, true, byte_no);
3176     }
3177     __ jmp(Done);
3178   }
3179 
3180   __ bind(notByte);
3181   __ cmpl(flags, ztos);
3182   __ jcc(Assembler::notEqual, notBool);
3183 
3184   // ztos
3185   {
3186     __ pop(ztos);
3187     if (!is_static) pop_and_check_object(obj);
3188     __ access_store_at(T_BOOLEAN, IN_HEAP, field, rax, noreg, noreg, noreg);
3189     if (!is_static && rc == may_rewrite) {
3190       patch_bytecode(Bytecodes::_fast_zputfield, bc, rbx, true, byte_no);
3191     }
3192     __ jmp(Done);
3193   }
3194 
3195   __ bind(notBool);
3196   __ cmpl(flags, atos);
3197   __ jcc(Assembler::notEqual, notObj);
3198 
3199   // atos
3200   {
3201     __ pop(atos);
3202     if (!is_static) pop_and_check_object(obj);
3203     // Store into the field
3204     do_oop_store(_masm, field, rax);
3205     if (!is_static && rc == may_rewrite) {
3206       patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
3207     }
3208     __ jmp(Done);
3209   }
3210 
3211   __ bind(notObj);
3212   __ cmpl(flags, itos);
3213   __ jcc(Assembler::notEqual, notInt);
3214 
3215   // itos
3216   {
3217     __ pop(itos);
3218     if (!is_static) pop_and_check_object(obj);
3219     __ access_store_at(T_INT, IN_HEAP, field, rax, noreg, noreg, noreg);
3220     if (!is_static && rc == may_rewrite) {
3221       patch_bytecode(Bytecodes::_fast_iputfield, bc, rbx, true, byte_no);
3222     }
3223     __ jmp(Done);
3224   }
3225 
3226   __ bind(notInt);
3227   __ cmpl(flags, ctos);
3228   __ jcc(Assembler::notEqual, notChar);
3229 
3230   // ctos
3231   {
3232     __ pop(ctos);
3233     if (!is_static) pop_and_check_object(obj);
3234     __ access_store_at(T_CHAR, IN_HEAP, field, rax, noreg, noreg, noreg);
3235     if (!is_static && rc == may_rewrite) {
3236       patch_bytecode(Bytecodes::_fast_cputfield, bc, rbx, true, byte_no);
3237     }
3238     __ jmp(Done);
3239   }
3240 
3241   __ bind(notChar);
3242   __ cmpl(flags, stos);
3243   __ jcc(Assembler::notEqual, notShort);
3244 
3245   // stos
3246   {
3247     __ pop(stos);
3248     if (!is_static) pop_and_check_object(obj);
3249     __ access_store_at(T_SHORT, IN_HEAP, field, rax, noreg, noreg, noreg);
3250     if (!is_static && rc == may_rewrite) {
3251       patch_bytecode(Bytecodes::_fast_sputfield, bc, rbx, true, byte_no);
3252     }
3253     __ jmp(Done);
3254   }
3255 
3256   __ bind(notShort);
3257   __ cmpl(flags, ltos);
3258   __ jcc(Assembler::notEqual, notLong);
3259 
3260   // ltos
3261   {
3262     __ pop(ltos);
3263     if (!is_static) pop_and_check_object(obj);
3264     // MO_RELAXED: generate atomic store for the case of volatile field (important for x86_32)
3265     __ access_store_at(T_LONG, IN_HEAP | MO_RELAXED, field, noreg /* ltos*/, noreg, noreg, noreg);
3266 #ifdef _LP64
3267     if (!is_static && rc == may_rewrite) {
3268       patch_bytecode(Bytecodes::_fast_lputfield, bc, rbx, true, byte_no);
3269     }
3270 #endif // _LP64
3271     __ jmp(Done);
3272   }
3273 
3274   __ bind(notLong);
3275   __ cmpl(flags, ftos);
3276   __ jcc(Assembler::notEqual, notFloat);
3277 
3278   // ftos
3279   {
3280     __ pop(ftos);
3281     if (!is_static) pop_and_check_object(obj);
3282     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg, noreg);
3283     if (!is_static && rc == may_rewrite) {
3284       patch_bytecode(Bytecodes::_fast_fputfield, bc, rbx, true, byte_no);
3285     }
3286     __ jmp(Done);
3287   }
3288 
3289   __ bind(notFloat);
3290 #ifdef ASSERT
3291   Label notDouble;
3292   __ cmpl(flags, dtos);
3293   __ jcc(Assembler::notEqual, notDouble);
3294 #endif
3295 
3296   // dtos
3297   {
3298     __ pop(dtos);
3299     if (!is_static) pop_and_check_object(obj);
3300     // MO_RELAXED: for the case of volatile field, in fact it adds no extra work for the underlying implementation
3301     __ access_store_at(T_DOUBLE, IN_HEAP | MO_RELAXED, field, noreg /* dtos */, noreg, noreg, noreg);
3302     if (!is_static && rc == may_rewrite) {
3303       patch_bytecode(Bytecodes::_fast_dputfield, bc, rbx, true, byte_no);
3304     }
3305   }
3306 
3307 #ifdef ASSERT
3308   __ jmp(Done);
3309 
3310   __ bind(notDouble);
3311   __ stop("Bad state");
3312 #endif
3313 
3314   __ bind(Done);
3315 }
3316 
3317 void TemplateTable::putfield(int byte_no) {
3318   putfield_or_static(byte_no, false);
3319 }
3320 
3321 void TemplateTable::nofast_putfield(int byte_no) {
3322   putfield_or_static(byte_no, false, may_not_rewrite);
3323 }
3324 
3325 void TemplateTable::putstatic(int byte_no) {
3326   putfield_or_static(byte_no, true);
3327 }
3328 
3329 void TemplateTable::jvmti_post_fast_field_mod() {
3330 
3331   const Register scratch = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
3332 
3333   if (JvmtiExport::can_post_field_modification()) {
3334     // Check to see if a field modification watch has been set before
3335     // we take the time to call into the VM.
3336     Label L2;
3337     __ mov32(scratch, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
3338     __ testl(scratch, scratch);
3339     __ jcc(Assembler::zero, L2);
3340     __ pop_ptr(rbx);                  // copy the object pointer from tos
3341     __ verify_oop(rbx);
3342     __ push_ptr(rbx);                 // put the object pointer back on tos
3343     // Save tos values before call_VM() clobbers them. Since we have
3344     // to do it for every data type, we use the saved values as the
3345     // jvalue object.
3346     switch (bytecode()) {          // load values into the jvalue object
3347     case Bytecodes::_fast_aputfield: __ push_ptr(rax); break;
3348     case Bytecodes::_fast_bputfield: // fall through
3349     case Bytecodes::_fast_zputfield: // fall through
3350     case Bytecodes::_fast_sputfield: // fall through
3351     case Bytecodes::_fast_cputfield: // fall through
3352     case Bytecodes::_fast_iputfield: __ push_i(rax); break;
3353     case Bytecodes::_fast_dputfield: __ push(dtos); break;
3354     case Bytecodes::_fast_fputfield: __ push(ftos); break;
3355     case Bytecodes::_fast_lputfield: __ push_l(rax); break;
3356 
3357     default:
3358       ShouldNotReachHere();
3359     }
3360     __ mov(scratch, rsp);             // points to jvalue on the stack
3361     // access constant pool cache entry
3362     LP64_ONLY(__ get_cache_entry_pointer_at_bcp(c_rarg2, rax, 1));
3363     NOT_LP64(__ get_cache_entry_pointer_at_bcp(rax, rdx, 1));
3364     __ verify_oop(rbx);
3365     // rbx: object pointer copied above
3366     // c_rarg2: cache entry pointer
3367     // c_rarg3: jvalue object on the stack
3368     LP64_ONLY(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), rbx, c_rarg2, c_rarg3));
3369     NOT_LP64(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), rbx, rax, rcx));
3370 
3371     switch (bytecode()) {             // restore tos values
3372     case Bytecodes::_fast_aputfield: __ pop_ptr(rax); break;
3373     case Bytecodes::_fast_bputfield: // fall through
3374     case Bytecodes::_fast_zputfield: // fall through
3375     case Bytecodes::_fast_sputfield: // fall through
3376     case Bytecodes::_fast_cputfield: // fall through
3377     case Bytecodes::_fast_iputfield: __ pop_i(rax); break;
3378     case Bytecodes::_fast_dputfield: __ pop(dtos); break;
3379     case Bytecodes::_fast_fputfield: __ pop(ftos); break;
3380     case Bytecodes::_fast_lputfield: __ pop_l(rax); break;
3381     default: break;
3382     }
3383     __ bind(L2);
3384   }
3385 }
3386 
3387 void TemplateTable::fast_storefield(TosState state) {
3388   transition(state, vtos);
3389 
3390   ByteSize base = ConstantPoolCache::base_offset();
3391 
3392   jvmti_post_fast_field_mod();
3393 
3394   // access constant pool cache
3395   __ get_cache_and_index_at_bcp(rcx, rbx, 1);
3396 
3397   // test for volatile with rdx but rdx is tos register for lputfield.
3398   __ movl(rdx, Address(rcx, rbx, Address::times_ptr,
3399                        in_bytes(base +
3400                                 ConstantPoolCacheEntry::flags_offset())));
3401 
3402   // replace index with field offset from cache entry
3403   __ movptr(rbx, Address(rcx, rbx, Address::times_ptr,
3404                          in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
3405 
3406   // [jk] not needed currently
3407   // volatile_barrier(Assembler::Membar_mask_bits(Assembler::LoadStore |
3408   //                                              Assembler::StoreStore));
3409 
3410   Label notVolatile, Done;
3411   __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift);
3412   __ andl(rdx, 0x1);
3413 
3414   // Get object from stack
3415   pop_and_check_object(rcx);
3416 
3417   // field address
3418   const Address field(rcx, rbx, Address::times_1);
3419 
3420   // Check for volatile store
3421   __ testl(rdx, rdx);
3422   __ jcc(Assembler::zero, notVolatile);
3423 
3424   fast_storefield_helper(field, rax);
3425   volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
3426                                                Assembler::StoreStore));
3427   __ jmp(Done);
3428   __ bind(notVolatile);
3429 
3430   fast_storefield_helper(field, rax);
3431 
3432   __ bind(Done);
3433 }
3434 
3435 void TemplateTable::fast_storefield_helper(Address field, Register rax) {
3436 
3437   // access field
3438   switch (bytecode()) {
3439   case Bytecodes::_fast_aputfield:
3440     do_oop_store(_masm, field, rax);
3441     break;
3442   case Bytecodes::_fast_lputfield:
3443 #ifdef _LP64
3444     __ access_store_at(T_LONG, IN_HEAP, field, noreg /* ltos */, noreg, noreg, noreg);
3445 #else
3446   __ stop("should not be rewritten");
3447 #endif
3448     break;
3449   case Bytecodes::_fast_iputfield:
3450     __ access_store_at(T_INT, IN_HEAP, field, rax, noreg, noreg, noreg);
3451     break;
3452   case Bytecodes::_fast_zputfield:
3453     __ access_store_at(T_BOOLEAN, IN_HEAP, field, rax, noreg, noreg, noreg);
3454     break;
3455   case Bytecodes::_fast_bputfield:
3456     __ access_store_at(T_BYTE, IN_HEAP, field, rax, noreg, noreg, noreg);
3457     break;
3458   case Bytecodes::_fast_sputfield:
3459     __ access_store_at(T_SHORT, IN_HEAP, field, rax, noreg, noreg, noreg);
3460     break;
3461   case Bytecodes::_fast_cputfield:
3462     __ access_store_at(T_CHAR, IN_HEAP, field, rax, noreg, noreg, noreg);
3463     break;
3464   case Bytecodes::_fast_fputfield:
3465     __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos*/, noreg, noreg, noreg);
3466     break;
3467   case Bytecodes::_fast_dputfield:
3468     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos*/, noreg, noreg, noreg);
3469     break;
3470   default:
3471     ShouldNotReachHere();
3472   }
3473 }
3474 
3475 void TemplateTable::fast_accessfield(TosState state) {
3476   transition(atos, state);
3477 
3478   // Do the JVMTI work here to avoid disturbing the register state below
3479   if (JvmtiExport::can_post_field_access()) {
3480     // Check to see if a field access watch has been set before we
3481     // take the time to call into the VM.
3482     Label L1;
3483     __ mov32(rcx, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
3484     __ testl(rcx, rcx);
3485     __ jcc(Assembler::zero, L1);
3486     // access constant pool cache entry
3487     LP64_ONLY(__ get_cache_entry_pointer_at_bcp(c_rarg2, rcx, 1));
3488     NOT_LP64(__ get_cache_entry_pointer_at_bcp(rcx, rdx, 1));
3489     __ verify_oop(rax);
3490     __ push_ptr(rax);  // save object pointer before call_VM() clobbers it
3491     LP64_ONLY(__ mov(c_rarg1, rax));
3492     // c_rarg1: object pointer copied above
3493     // c_rarg2: cache entry pointer
3494     LP64_ONLY(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access), c_rarg1, c_rarg2));
3495     NOT_LP64(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access), rax, rcx));
3496     __ pop_ptr(rax); // restore object pointer
3497     __ bind(L1);
3498   }
3499 
3500   // access constant pool cache
3501   __ get_cache_and_index_at_bcp(rcx, rbx, 1);
3502   // replace index with field offset from cache entry
3503   // [jk] not needed currently
3504   // __ movl(rdx, Address(rcx, rbx, Address::times_8,
3505   //                      in_bytes(ConstantPoolCache::base_offset() +
3506   //                               ConstantPoolCacheEntry::flags_offset())));
3507   // __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift);
3508   // __ andl(rdx, 0x1);
3509   //
3510   __ movptr(rbx, Address(rcx, rbx, Address::times_ptr,
3511                          in_bytes(ConstantPoolCache::base_offset() +
3512                                   ConstantPoolCacheEntry::f2_offset())));
3513 
3514   // rax: object
3515   __ verify_oop(rax);
3516   __ null_check(rax);
3517   Address field(rax, rbx, Address::times_1);
3518 
3519   // access field
3520   switch (bytecode()) {
3521   case Bytecodes::_fast_agetfield:
3522     do_oop_load(_masm, field, rax);
3523     __ verify_oop(rax);
3524     break;
3525   case Bytecodes::_fast_lgetfield:
3526 #ifdef _LP64
3527     __ access_load_at(T_LONG, IN_HEAP, noreg /* ltos */, field, noreg, noreg);
3528 #else
3529   __ stop("should not be rewritten");
3530 #endif
3531     break;
3532   case Bytecodes::_fast_igetfield:
3533     __ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
3534     break;
3535   case Bytecodes::_fast_bgetfield:
3536     __ access_load_at(T_BYTE, IN_HEAP, rax, field, noreg, noreg);
3537     break;
3538   case Bytecodes::_fast_sgetfield:
3539     __ access_load_at(T_SHORT, IN_HEAP, rax, field, noreg, noreg);
3540     break;
3541   case Bytecodes::_fast_cgetfield:
3542     __ access_load_at(T_CHAR, IN_HEAP, rax, field, noreg, noreg);
3543     break;
3544   case Bytecodes::_fast_fgetfield:
3545     __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
3546     break;
3547   case Bytecodes::_fast_dgetfield:
3548     __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* dtos */, field, noreg, noreg);
3549     break;
3550   default:
3551     ShouldNotReachHere();
3552   }
3553   // [jk] not needed currently
3554   //   Label notVolatile;
3555   //   __ testl(rdx, rdx);
3556   //   __ jcc(Assembler::zero, notVolatile);
3557   //   __ membar(Assembler::LoadLoad);
3558   //   __ bind(notVolatile);
3559 }
3560 
3561 void TemplateTable::fast_xaccess(TosState state) {
3562   transition(vtos, state);
3563 
3564   // get receiver
3565   __ movptr(rax, aaddress(0));
3566   // access constant pool cache
3567   __ get_cache_and_index_at_bcp(rcx, rdx, 2);
3568   __ movptr(rbx,
3569             Address(rcx, rdx, Address::times_ptr,
3570                     in_bytes(ConstantPoolCache::base_offset() +
3571                              ConstantPoolCacheEntry::f2_offset())));
3572   // make sure exception is reported in correct bcp range (getfield is
3573   // next instruction)
3574   __ increment(rbcp);
3575   __ null_check(rax);
3576   const Address field = Address(rax, rbx, Address::times_1, 0*wordSize);
3577   switch (state) {
3578   case itos:
3579     __ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
3580     break;
3581   case atos:
3582     do_oop_load(_masm, field, rax);
3583     __ verify_oop(rax);
3584     break;
3585   case ftos:
3586     __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg);
3587     break;
3588   default:
3589     ShouldNotReachHere();
3590   }
3591 
3592   // [jk] not needed currently
3593   // Label notVolatile;
3594   // __ movl(rdx, Address(rcx, rdx, Address::times_8,
3595   //                      in_bytes(ConstantPoolCache::base_offset() +
3596   //                               ConstantPoolCacheEntry::flags_offset())));
3597   // __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift);
3598   // __ testl(rdx, 0x1);
3599   // __ jcc(Assembler::zero, notVolatile);
3600   // __ membar(Assembler::LoadLoad);
3601   // __ bind(notVolatile);
3602 
3603   __ decrement(rbcp);
3604 }
3605 
3606 //-----------------------------------------------------------------------------
3607 // Calls
3608 
3609 void TemplateTable::prepare_invoke(int byte_no,
3610                                    Register method,  // linked method (or i-klass)
3611                                    Register index,   // itable index, MethodType, etc.
3612                                    Register recv,    // if caller wants to see it
3613                                    Register flags    // if caller wants to test it
3614                                    ) {
3615   // determine flags
3616   const Bytecodes::Code code = bytecode();
3617   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3618   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3619   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3620   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3621   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3622   const bool load_receiver       = (recv  != noreg);
3623   const bool save_flags          = (flags != noreg);
3624   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3625   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3626   assert(flags == noreg || flags == rdx, "");
3627   assert(recv  == noreg || recv  == rcx, "");
3628 
3629   // setup registers & access constant pool cache
3630   if (recv  == noreg)  recv  = rcx;
3631   if (flags == noreg)  flags = rdx;
3632   assert_different_registers(method, index, recv, flags);
3633 
3634   // save 'interpreter return address'
3635   __ save_bcp();
3636 
3637   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3638 
3639   // maybe push appendix to arguments (just before return address)
3640   if (is_invokehandle) {
3641     Label L_no_push;
3642     __ testl(flags, (1 << ConstantPoolCacheEntry::has_appendix_shift));
3643     __ jcc(Assembler::zero, L_no_push);
3644     // Push the appendix as a trailing parameter.
3645     // This must be done before we get the receiver,
3646     // since the parameter_size includes it.
3647     __ push(rbx);
3648     __ mov(rbx, index);
3649     __ load_resolved_reference_at_index(index, rbx);
3650     __ pop(rbx);
3651     __ push(index);  // push appendix (MethodType, CallSite, etc.)
3652     __ bind(L_no_push);
3653   }
3654 
3655   // load receiver if needed (after appendix is pushed so parameter size is correct)
3656   // Note: no return address pushed yet
3657   if (load_receiver) {
3658     __ movl(recv, flags);
3659     __ andl(recv, ConstantPoolCacheEntry::parameter_size_mask);
3660     const int no_return_pc_pushed_yet = -1;  // argument slot correction before we push return address
3661     const int receiver_is_at_end      = -1;  // back off one slot to get receiver
3662     Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
3663     __ movptr(recv, recv_addr);
3664     __ verify_oop(recv);
3665   }
3666 
3667   if (save_flags) {
3668     __ movl(rbcp, flags);
3669   }
3670 
3671   // compute return type
3672   __ shrl(flags, ConstantPoolCacheEntry::tos_state_shift);
3673   // Make sure we don't need to mask flags after the above shift
3674   ConstantPoolCacheEntry::verify_tos_state_shift();
3675   // load return address
3676   {
3677     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3678     ExternalAddress table(table_addr);
3679 #ifdef _LP64
3680     __ lea(rscratch1, table);
3681     __ movptr(flags, Address(rscratch1, flags, Address::times_ptr));
3682 #else
3683     __ movptr(flags, ArrayAddress(table, Address(noreg, flags, Address::times_ptr)));
3684 #endif // _LP64
3685   }
3686 
3687   // push return address
3688   __ push(flags);
3689 
3690   // Restore flags value from the constant pool cache, and restore rsi
3691   // for later null checks.  r13 is the bytecode pointer
3692   if (save_flags) {
3693     __ movl(flags, rbcp);
3694     __ restore_bcp();
3695   }
3696 }
3697 
3698 void TemplateTable::invokevirtual_helper(Register index,
3699                                          Register recv,
3700                                          Register flags) {
3701   // Uses temporary registers rax, rdx
3702   assert_different_registers(index, recv, rax, rdx);
3703   assert(index == rbx, "");
3704   assert(recv  == rcx, "");
3705 
3706   // Test for an invoke of a final method
3707   Label notFinal;
3708   __ movl(rax, flags);
3709   __ andl(rax, (1 << ConstantPoolCacheEntry::is_vfinal_shift));
3710   __ jcc(Assembler::zero, notFinal);
3711 
3712   const Register method = index;  // method must be rbx
3713   assert(method == rbx,
3714          "Method* must be rbx for interpreter calling convention");
3715 
3716   // do the call - the index is actually the method to call
3717   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3718 
3719   // It's final, need a null check here!
3720   __ null_check(recv);
3721 
3722   // profile this call
3723   __ profile_final_call(rax);
3724   __ profile_arguments_type(rax, method, rbcp, true);
3725 
3726   __ jump_from_interpreted(method, rax);
3727 
3728   __ bind(notFinal);
3729 
3730   // get receiver klass
3731   __ load_klass(rax, recv, rscratch1);
3732 
3733   // profile this call
3734   __ profile_virtual_call(rax, rlocals, rdx);
3735   // get target Method* & entry point
3736   __ lookup_virtual_method(rax, index, method);
3737 
3738   __ profile_arguments_type(rdx, method, rbcp, true);
3739   __ jump_from_interpreted(method, rdx);
3740 }
3741 
3742 void TemplateTable::invokevirtual(int byte_no) {
3743   transition(vtos, vtos);
3744   assert(byte_no == f2_byte, "use this argument");
3745   prepare_invoke(byte_no,
3746                  rbx,    // method or vtable index
3747                  noreg,  // unused itable index
3748                  rcx, rdx); // recv, flags
3749 
3750   // rbx: index
3751   // rcx: receiver
3752   // rdx: flags
3753 
3754   invokevirtual_helper(rbx, rcx, rdx);
3755 }
3756 
3757 void TemplateTable::invokespecial(int byte_no) {
3758   transition(vtos, vtos);
3759   assert(byte_no == f1_byte, "use this argument");
3760   prepare_invoke(byte_no, rbx, noreg,  // get f1 Method*
3761                  rcx);  // get receiver also for null check
3762   __ verify_oop(rcx);
3763   __ null_check(rcx);
3764   // do the call
3765   __ profile_call(rax);
3766   __ profile_arguments_type(rax, rbx, rbcp, false);
3767   __ jump_from_interpreted(rbx, rax);
3768 }
3769 
3770 void TemplateTable::invokestatic(int byte_no) {
3771   transition(vtos, vtos);
3772   assert(byte_no == f1_byte, "use this argument");
3773   prepare_invoke(byte_no, rbx);  // get f1 Method*
3774   // do the call
3775   __ profile_call(rax);
3776   __ profile_arguments_type(rax, rbx, rbcp, false);
3777   __ jump_from_interpreted(rbx, rax);
3778 }
3779 
3780 
3781 void TemplateTable::fast_invokevfinal(int byte_no) {
3782   transition(vtos, vtos);
3783   assert(byte_no == f2_byte, "use this argument");
3784   __ stop("fast_invokevfinal not used on x86");
3785 }
3786 
3787 
3788 void TemplateTable::invokeinterface(int byte_no) {
3789   transition(vtos, vtos);
3790   assert(byte_no == f1_byte, "use this argument");
3791   prepare_invoke(byte_no, rax, rbx,  // get f1 Klass*, f2 Method*
3792                  rcx, rdx); // recv, flags
3793 
3794   // rax: reference klass (from f1) if interface method
3795   // rbx: method (from f2)
3796   // rcx: receiver
3797   // rdx: flags
3798 
3799   // First check for Object case, then private interface method,
3800   // then regular interface method.
3801 
3802   // Special case of invokeinterface called for virtual method of
3803   // java.lang.Object.  See cpCache.cpp for details.
3804   Label notObjectMethod;
3805   __ movl(rlocals, rdx);
3806   __ andl(rlocals, (1 << ConstantPoolCacheEntry::is_forced_virtual_shift));
3807   __ jcc(Assembler::zero, notObjectMethod);
3808   invokevirtual_helper(rbx, rcx, rdx);
3809   // no return from above
3810   __ bind(notObjectMethod);
3811 
3812   Label no_such_interface; // for receiver subtype check
3813   Register recvKlass; // used for exception processing
3814 
3815   // Check for private method invocation - indicated by vfinal
3816   Label notVFinal;
3817   __ movl(rlocals, rdx);
3818   __ andl(rlocals, (1 << ConstantPoolCacheEntry::is_vfinal_shift));
3819   __ jcc(Assembler::zero, notVFinal);
3820 
3821   // Get receiver klass into rlocals - also a null check
3822   __ load_klass(rlocals, rcx, rscratch1);
3823 
3824   Label subtype;
3825   __ check_klass_subtype(rlocals, rax, rbcp, subtype);
3826   // If we get here the typecheck failed
3827   recvKlass = rdx;
3828   __ mov(recvKlass, rlocals); // shuffle receiver class for exception use
3829   __ jmp(no_such_interface);
3830 
3831   __ bind(subtype);
3832 
3833   // do the call - rbx is actually the method to call
3834 
3835   __ profile_final_call(rdx);
3836   __ profile_arguments_type(rdx, rbx, rbcp, true);
3837 
3838   __ jump_from_interpreted(rbx, rdx);
3839   // no return from above
3840   __ bind(notVFinal);
3841 
3842   // Get receiver klass into rdx - also a null check
3843   __ restore_locals();  // restore r14
3844   __ load_klass(rdx, rcx, rscratch1);
3845 
3846   Label no_such_method;
3847 
3848   // Preserve method for throw_AbstractMethodErrorVerbose.
3849   __ mov(rcx, rbx);
3850   // Receiver subtype check against REFC.
3851   // Superklass in rax. Subklass in rdx. Blows rcx, rdi.
3852   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3853                              rdx, rax, noreg,
3854                              // outputs: scan temp. reg, scan temp. reg
3855                              rbcp, rlocals,
3856                              no_such_interface,
3857                              /*return_method=*/false);
3858 
3859   // profile this call
3860   __ restore_bcp(); // rbcp was destroyed by receiver type check
3861   __ profile_virtual_call(rdx, rbcp, rlocals);
3862 
3863   // Get declaring interface class from method, and itable index
3864   __ load_method_holder(rax, rbx);
3865   __ movl(rbx, Address(rbx, Method::itable_index_offset()));
3866   __ subl(rbx, Method::itable_index_max);
3867   __ negl(rbx);
3868 
3869   // Preserve recvKlass for throw_AbstractMethodErrorVerbose.
3870   __ mov(rlocals, rdx);
3871   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3872                              rlocals, rax, rbx,
3873                              // outputs: method, scan temp. reg
3874                              rbx, rbcp,
3875                              no_such_interface);
3876 
3877   // rbx: Method* to call
3878   // rcx: receiver
3879   // Check for abstract method error
3880   // Note: This should be done more efficiently via a throw_abstract_method_error
3881   //       interpreter entry point and a conditional jump to it in case of a null
3882   //       method.
3883   __ testptr(rbx, rbx);
3884   __ jcc(Assembler::zero, no_such_method);
3885 
3886   __ profile_arguments_type(rdx, rbx, rbcp, true);
3887 
3888   // do the call
3889   // rcx: receiver
3890   // rbx,: Method*
3891   __ jump_from_interpreted(rbx, rdx);
3892   __ should_not_reach_here();
3893 
3894   // exception handling code follows...
3895   // note: must restore interpreter registers to canonical
3896   //       state for exception handling to work correctly!
3897 
3898   __ bind(no_such_method);
3899   // throw exception
3900   __ pop(rbx);           // pop return address (pushed by prepare_invoke)
3901   __ restore_bcp();      // rbcp must be correct for exception handler   (was destroyed)
3902   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3903   // Pass arguments for generating a verbose error message.
3904 #ifdef _LP64
3905   recvKlass = c_rarg1;
3906   Register method    = c_rarg2;
3907   if (recvKlass != rdx) { __ movq(recvKlass, rdx); }
3908   if (method != rcx)    { __ movq(method, rcx);    }
3909 #else
3910   recvKlass = rdx;
3911   Register method    = rcx;
3912 #endif
3913   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodErrorVerbose),
3914              recvKlass, method);
3915   // The call_VM checks for exception, so we should never return here.
3916   __ should_not_reach_here();
3917 
3918   __ bind(no_such_interface);
3919   // throw exception
3920   __ pop(rbx);           // pop return address (pushed by prepare_invoke)
3921   __ restore_bcp();      // rbcp must be correct for exception handler   (was destroyed)
3922   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3923   // Pass arguments for generating a verbose error message.
3924   LP64_ONLY( if (recvKlass != rdx) { __ movq(recvKlass, rdx); } )
3925   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_IncompatibleClassChangeErrorVerbose),
3926              recvKlass, rax);
3927   // the call_VM checks for exception, so we should never return here.
3928   __ should_not_reach_here();
3929 }
3930 
3931 void TemplateTable::invokehandle(int byte_no) {
3932   transition(vtos, vtos);
3933   assert(byte_no == f1_byte, "use this argument");
3934   const Register rbx_method = rbx;
3935   const Register rax_mtype  = rax;
3936   const Register rcx_recv   = rcx;
3937   const Register rdx_flags  = rdx;
3938 
3939   prepare_invoke(byte_no, rbx_method, rax_mtype, rcx_recv);
3940   __ verify_method_ptr(rbx_method);
3941   __ verify_oop(rcx_recv);
3942   __ null_check(rcx_recv);
3943 
3944   // rax: MethodType object (from cpool->resolved_references[f1], if necessary)
3945   // rbx: MH.invokeExact_MT method (from f2)
3946 
3947   // Note:  rax_mtype is already pushed (if necessary) by prepare_invoke
3948 
3949   // FIXME: profile the LambdaForm also
3950   __ profile_final_call(rax);
3951   __ profile_arguments_type(rdx, rbx_method, rbcp, true);
3952 
3953   __ jump_from_interpreted(rbx_method, rdx);
3954 }
3955 
3956 void TemplateTable::invokedynamic(int byte_no) {
3957   transition(vtos, vtos);
3958   assert(byte_no == f1_byte, "use this argument");
3959 
3960   const Register rbx_method   = rbx;
3961   const Register rax_callsite = rax;
3962 
3963   load_invokedynamic_entry(rbx_method);
3964   // rax: CallSite object (from cpool->resolved_references[f1])
3965   // rbx: MH.linkToCallSite method (from f2)
3966 
3967   // Note:  rax_callsite is already pushed by prepare_invoke
3968 
3969   // %%% should make a type profile for any invokedynamic that takes a ref argument
3970   // profile this call
3971   __ profile_call(rbcp);
3972   __ profile_arguments_type(rdx, rbx_method, rbcp, false);
3973 
3974   __ verify_oop(rax_callsite);
3975 
3976   __ jump_from_interpreted(rbx_method, rdx);
3977 }
3978 
3979 //-----------------------------------------------------------------------------
3980 // Allocation
3981 
3982 void TemplateTable::_new() {
3983   transition(vtos, atos);
3984   __ get_unsigned_2_byte_index_at_bcp(rdx, 1);
3985   Label slow_case;
3986   Label slow_case_no_pop;
3987   Label done;
3988   Label initialize_header;
3989 
3990   __ get_cpool_and_tags(rcx, rax);
3991 
3992   // Make sure the class we're about to instantiate has been resolved.
3993   // This is done before loading InstanceKlass to be consistent with the order
3994   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3995   const int tags_offset = Array<u1>::base_offset_in_bytes();
3996   __ cmpb(Address(rax, rdx, Address::times_1, tags_offset), JVM_CONSTANT_Class);
3997   __ jcc(Assembler::notEqual, slow_case_no_pop);
3998 
3999   // get InstanceKlass
4000   __ load_resolved_klass_at_index(rcx, rcx, rdx);
4001   __ push(rcx);  // save the contexts of klass for initializing the header
4002 
4003   // make sure klass is initialized & doesn't have finalizer
4004   // make sure klass is fully initialized
4005   __ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
4006   __ jcc(Assembler::notEqual, slow_case);
4007 
4008   // get instance_size in InstanceKlass (scaled to a count of bytes)
4009   __ movl(rdx, Address(rcx, Klass::layout_helper_offset()));
4010   // test to see if it has a finalizer or is malformed in some way
4011   __ testl(rdx, Klass::_lh_instance_slow_path_bit);
4012   __ jcc(Assembler::notZero, slow_case);
4013 
4014   // Allocate the instance:
4015   //  If TLAB is enabled:
4016   //    Try to allocate in the TLAB.
4017   //    If fails, go to the slow path.
4018   //    Initialize the allocation.
4019   //    Exit.
4020   //
4021   //  Go to slow path.
4022 
4023   const Register thread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
4024 
4025   if (UseTLAB) {
4026     NOT_LP64(__ get_thread(thread);)
4027     __ tlab_allocate(thread, rax, rdx, 0, rcx, rbx, slow_case);
4028     if (ZeroTLAB) {
4029       // the fields have been already cleared
4030       __ jmp(initialize_header);
4031     }
4032 
4033     // The object is initialized before the header.  If the object size is
4034     // zero, go directly to the header initialization.
4035     __ decrement(rdx, sizeof(oopDesc));
4036     __ jcc(Assembler::zero, initialize_header);
4037 
4038     // Initialize topmost object field, divide rdx by 8, check if odd and
4039     // test if zero.
4040     __ xorl(rcx, rcx);    // use zero reg to clear memory (shorter code)
4041     __ shrl(rdx, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd
4042 
4043     // rdx must have been multiple of 8
4044 #ifdef ASSERT
4045     // make sure rdx was multiple of 8
4046     Label L;
4047     // Ignore partial flag stall after shrl() since it is debug VM
4048     __ jcc(Assembler::carryClear, L);
4049     __ stop("object size is not multiple of 2 - adjust this code");
4050     __ bind(L);
4051     // rdx must be > 0, no extra check needed here
4052 #endif
4053 
4054     // initialize remaining object fields: rdx was a multiple of 8
4055     { Label loop;
4056     __ bind(loop);
4057     __ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 1*oopSize), rcx);
4058     NOT_LP64(__ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 2*oopSize), rcx));
4059     __ decrement(rdx);
4060     __ jcc(Assembler::notZero, loop);
4061     }
4062 
4063     // initialize object header only.
4064     __ bind(initialize_header);
4065     __ movptr(Address(rax, oopDesc::mark_offset_in_bytes()),
4066               (intptr_t)markWord::prototype().value()); // header
4067     __ pop(rcx);   // get saved klass back in the register.
4068 #ifdef _LP64
4069     __ xorl(rsi, rsi); // use zero reg to clear memory (shorter code)
4070     __ store_klass_gap(rax, rsi);  // zero klass gap for compressed oops
4071 #endif
4072     __ store_klass(rax, rcx, rscratch1);  // klass
4073 
4074     {
4075       SkipIfEqual skip_if(_masm, &DTraceAllocProbes, 0, rscratch1);
4076       // Trigger dtrace event for fastpath
4077       __ push(atos);
4078       __ call_VM_leaf(
4079            CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), rax);
4080       __ pop(atos);
4081     }
4082 
4083     __ jmp(done);
4084   }
4085 
4086   // slow case
4087   __ bind(slow_case);
4088   __ pop(rcx);   // restore stack pointer to what it was when we came in.
4089   __ bind(slow_case_no_pop);
4090 
4091   Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rax);
4092   Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
4093 
4094   __ get_constant_pool(rarg1);
4095   __ get_unsigned_2_byte_index_at_bcp(rarg2, 1);
4096   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), rarg1, rarg2);
4097    __ verify_oop(rax);
4098 
4099   // continue
4100   __ bind(done);
4101 }
4102 
4103 void TemplateTable::newarray() {
4104   transition(itos, atos);
4105   Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rdx);
4106   __ load_unsigned_byte(rarg1, at_bcp(1));
4107   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
4108           rarg1, rax);
4109 }
4110 
4111 void TemplateTable::anewarray() {
4112   transition(itos, atos);
4113 
4114   Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rcx);
4115   Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
4116 
4117   __ get_unsigned_2_byte_index_at_bcp(rarg2, 1);
4118   __ get_constant_pool(rarg1);
4119   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
4120           rarg1, rarg2, rax);
4121 }
4122 
4123 void TemplateTable::arraylength() {
4124   transition(atos, itos);
4125   __ movl(rax, Address(rax, arrayOopDesc::length_offset_in_bytes()));
4126 }
4127 
4128 void TemplateTable::checkcast() {
4129   transition(atos, atos);
4130   Label done, is_null, ok_is_subtype, quicked, resolved;
4131   __ testptr(rax, rax); // object is in rax
4132   __ jcc(Assembler::zero, is_null);
4133 
4134   // Get cpool & tags index
4135   __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
4136   __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
4137   // See if bytecode has already been quicked
4138   __ cmpb(Address(rdx, rbx,
4139                   Address::times_1,
4140                   Array<u1>::base_offset_in_bytes()),
4141           JVM_CONSTANT_Class);
4142   __ jcc(Assembler::equal, quicked);
4143   __ push(atos); // save receiver for result, and for GC
4144   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4145 
4146   // vm_result_2 has metadata result
4147 #ifndef _LP64
4148   // borrow rdi from locals
4149   __ get_thread(rdi);
4150   __ get_vm_result_2(rax, rdi);
4151   __ restore_locals();
4152 #else
4153   __ get_vm_result_2(rax, r15_thread);
4154 #endif
4155 
4156   __ pop_ptr(rdx); // restore receiver
4157   __ jmpb(resolved);
4158 
4159   // Get superklass in rax and subklass in rbx
4160   __ bind(quicked);
4161   __ mov(rdx, rax); // Save object in rdx; rax needed for subtype check
4162   __ load_resolved_klass_at_index(rax, rcx, rbx);
4163 
4164   __ bind(resolved);
4165   __ load_klass(rbx, rdx, rscratch1);
4166 
4167   // Generate subtype check.  Blows rcx, rdi.  Object in rdx.
4168   // Superklass in rax.  Subklass in rbx.
4169   __ gen_subtype_check(rbx, ok_is_subtype);
4170 
4171   // Come here on failure
4172   __ push_ptr(rdx);
4173   // object is at TOS
4174   __ jump(ExternalAddress(Interpreter::_throw_ClassCastException_entry));
4175 
4176   // Come here on success
4177   __ bind(ok_is_subtype);
4178   __ mov(rax, rdx); // Restore object in rdx
4179 
4180   // Collect counts on whether this check-cast sees nulls a lot or not.
4181   if (ProfileInterpreter) {
4182     __ jmp(done);
4183     __ bind(is_null);
4184     __ profile_null_seen(rcx);
4185   } else {
4186     __ bind(is_null);   // same as 'done'
4187   }
4188   __ bind(done);
4189 }
4190 
4191 void TemplateTable::instanceof() {
4192   transition(atos, itos);
4193   Label done, is_null, ok_is_subtype, quicked, resolved;
4194   __ testptr(rax, rax);
4195   __ jcc(Assembler::zero, is_null);
4196 
4197   // Get cpool & tags index
4198   __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
4199   __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
4200   // See if bytecode has already been quicked
4201   __ cmpb(Address(rdx, rbx,
4202                   Address::times_1,
4203                   Array<u1>::base_offset_in_bytes()),
4204           JVM_CONSTANT_Class);
4205   __ jcc(Assembler::equal, quicked);
4206 
4207   __ push(atos); // save receiver for result, and for GC
4208   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4209   // vm_result_2 has metadata result
4210 
4211 #ifndef _LP64
4212   // borrow rdi from locals
4213   __ get_thread(rdi);
4214   __ get_vm_result_2(rax, rdi);
4215   __ restore_locals();
4216 #else
4217   __ get_vm_result_2(rax, r15_thread);
4218 #endif
4219 
4220   __ pop_ptr(rdx); // restore receiver
4221   __ verify_oop(rdx);
4222   __ load_klass(rdx, rdx, rscratch1);
4223   __ jmpb(resolved);
4224 
4225   // Get superklass in rax and subklass in rdx
4226   __ bind(quicked);
4227   __ load_klass(rdx, rax, rscratch1);
4228   __ load_resolved_klass_at_index(rax, rcx, rbx);
4229 
4230   __ bind(resolved);
4231 
4232   // Generate subtype check.  Blows rcx, rdi
4233   // Superklass in rax.  Subklass in rdx.
4234   __ gen_subtype_check(rdx, ok_is_subtype);
4235 
4236   // Come here on failure
4237   __ xorl(rax, rax);
4238   __ jmpb(done);
4239   // Come here on success
4240   __ bind(ok_is_subtype);
4241   __ movl(rax, 1);
4242 
4243   // Collect counts on whether this test sees nulls a lot or not.
4244   if (ProfileInterpreter) {
4245     __ jmp(done);
4246     __ bind(is_null);
4247     __ profile_null_seen(rcx);
4248   } else {
4249     __ bind(is_null);   // same as 'done'
4250   }
4251   __ bind(done);
4252   // rax = 0: obj == nullptr or  obj is not an instanceof the specified klass
4253   // rax = 1: obj != nullptr and obj is     an instanceof the specified klass
4254 }
4255 
4256 
4257 //----------------------------------------------------------------------------------------------------
4258 // Breakpoints
4259 void TemplateTable::_breakpoint() {
4260   // Note: We get here even if we are single stepping..
4261   // jbug insists on setting breakpoints at every bytecode
4262   // even if we are in single step mode.
4263 
4264   transition(vtos, vtos);
4265 
4266   Register rarg = LP64_ONLY(c_rarg1) NOT_LP64(rcx);
4267 
4268   // get the unpatched byte code
4269   __ get_method(rarg);
4270   __ call_VM(noreg,
4271              CAST_FROM_FN_PTR(address,
4272                               InterpreterRuntime::get_original_bytecode_at),
4273              rarg, rbcp);
4274   __ mov(rbx, rax);  // why?
4275 
4276   // post the breakpoint event
4277   __ get_method(rarg);
4278   __ call_VM(noreg,
4279              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
4280              rarg, rbcp);
4281 
4282   // complete the execution of original bytecode
4283   __ dispatch_only_normal(vtos);
4284 }
4285 
4286 //-----------------------------------------------------------------------------
4287 // Exceptions
4288 
4289 void TemplateTable::athrow() {
4290   transition(atos, vtos);
4291   __ null_check(rax);
4292   __ jump(ExternalAddress(Interpreter::throw_exception_entry()));
4293 }
4294 
4295 //-----------------------------------------------------------------------------
4296 // Synchronization
4297 //
4298 // Note: monitorenter & exit are symmetric routines; which is reflected
4299 //       in the assembly code structure as well
4300 //
4301 // Stack layout:
4302 //
4303 // [expressions  ] <--- rsp               = expression stack top
4304 // ..
4305 // [expressions  ]
4306 // [monitor entry] <--- monitor block top = expression stack bot
4307 // ..
4308 // [monitor entry]
4309 // [frame data   ] <--- monitor block bot
4310 // ...
4311 // [saved rbp    ] <--- rbp
4312 void TemplateTable::monitorenter() {
4313   transition(atos, vtos);
4314 
4315   // check for null object
4316   __ null_check(rax);
4317 
4318   const Address monitor_block_top(
4319         rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4320   const Address monitor_block_bot(
4321         rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
4322   const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
4323 
4324   Label allocated;
4325 
4326   Register rtop = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
4327   Register rbot = LP64_ONLY(c_rarg2) NOT_LP64(rbx);
4328   Register rmon = LP64_ONLY(c_rarg1) NOT_LP64(rdx);
4329 
4330   // initialize entry pointer
4331   __ xorl(rmon, rmon); // points to free slot or null
4332 
4333   // find a free slot in the monitor block (result in rmon)
4334   {
4335     Label entry, loop, exit;
4336     __ movptr(rtop, monitor_block_top); // points to current entry,
4337                                         // starting with top-most entry
4338     __ lea(rbot, monitor_block_bot);    // points to word before bottom
4339                                         // of monitor block
4340     __ jmpb(entry);
4341 
4342     __ bind(loop);
4343     // check if current entry is used
4344     __ cmpptr(Address(rtop, BasicObjectLock::obj_offset()), NULL_WORD);
4345     // if not used then remember entry in rmon
4346     __ cmovptr(Assembler::equal, rmon, rtop);   // cmov => cmovptr
4347     // check if current entry is for same object
4348     __ cmpptr(rax, Address(rtop, BasicObjectLock::obj_offset()));
4349     // if same object then stop searching
4350     __ jccb(Assembler::equal, exit);
4351     // otherwise advance to next entry
4352     __ addptr(rtop, entry_size);
4353     __ bind(entry);
4354     // check if bottom reached
4355     __ cmpptr(rtop, rbot);
4356     // if not at bottom then check this entry
4357     __ jcc(Assembler::notEqual, loop);
4358     __ bind(exit);
4359   }
4360 
4361   __ testptr(rmon, rmon); // check if a slot has been found
4362   __ jcc(Assembler::notZero, allocated); // if found, continue with that one
4363 
4364   // allocate one if there's no free slot
4365   {
4366     Label entry, loop;
4367     // 1. compute new pointers          // rsp: old expression stack top
4368     __ movptr(rmon, monitor_block_bot); // rmon: old expression stack bottom
4369     __ subptr(rsp, entry_size);         // move expression stack top
4370     __ subptr(rmon, entry_size);        // move expression stack bottom
4371     __ mov(rtop, rsp);                  // set start value for copy loop
4372     __ movptr(monitor_block_bot, rmon); // set new monitor block bottom
4373     __ jmp(entry);
4374     // 2. move expression stack contents
4375     __ bind(loop);
4376     __ movptr(rbot, Address(rtop, entry_size)); // load expression stack
4377                                                 // word from old location
4378     __ movptr(Address(rtop, 0), rbot);          // and store it at new location
4379     __ addptr(rtop, wordSize);                  // advance to next word
4380     __ bind(entry);
4381     __ cmpptr(rtop, rmon);                      // check if bottom reached
4382     __ jcc(Assembler::notEqual, loop);          // if not at bottom then
4383                                                 // copy next word
4384   }
4385 
4386   // call run-time routine
4387   // rmon: points to monitor entry
4388   __ bind(allocated);
4389 
4390   // Increment bcp to point to the next bytecode, so exception
4391   // handling for async. exceptions work correctly.
4392   // The object has already been popped from the stack, so the
4393   // expression stack looks correct.
4394   __ increment(rbcp);
4395 
4396   // store object
4397   __ movptr(Address(rmon, BasicObjectLock::obj_offset()), rax);
4398   __ lock_object(rmon);
4399 
4400   // check to make sure this monitor doesn't cause stack overflow after locking
4401   __ save_bcp();  // in case of exception
4402   __ generate_stack_overflow_check(0);
4403 
4404   // The bcp has already been incremented. Just need to dispatch to
4405   // next instruction.
4406   __ dispatch_next(vtos);
4407 }
4408 
4409 void TemplateTable::monitorexit() {
4410   transition(atos, vtos);
4411 
4412   // check for null object
4413   __ null_check(rax);
4414 
4415   const Address monitor_block_top(
4416         rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4417   const Address monitor_block_bot(
4418         rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
4419   const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
4420 
4421   Register rtop = LP64_ONLY(c_rarg1) NOT_LP64(rdx);
4422   Register rbot = LP64_ONLY(c_rarg2) NOT_LP64(rbx);
4423 
4424   Label found;
4425 
4426   // find matching slot
4427   {
4428     Label entry, loop;
4429     __ movptr(rtop, monitor_block_top); // points to current entry,
4430                                         // starting with top-most entry
4431     __ lea(rbot, monitor_block_bot);    // points to word before bottom
4432                                         // of monitor block
4433     __ jmpb(entry);
4434 
4435     __ bind(loop);
4436     // check if current entry is for same object
4437     __ cmpptr(rax, Address(rtop, BasicObjectLock::obj_offset()));
4438     // if same object then stop searching
4439     __ jcc(Assembler::equal, found);
4440     // otherwise advance to next entry
4441     __ addptr(rtop, entry_size);
4442     __ bind(entry);
4443     // check if bottom reached
4444     __ cmpptr(rtop, rbot);
4445     // if not at bottom then check this entry
4446     __ jcc(Assembler::notEqual, loop);
4447   }
4448 
4449   // error handling. Unlocking was not block-structured
4450   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4451                    InterpreterRuntime::throw_illegal_monitor_state_exception));
4452   __ should_not_reach_here();
4453 
4454   // call run-time routine
4455   __ bind(found);
4456   __ push_ptr(rax); // make sure object is on stack (contract with oopMaps)
4457   __ unlock_object(rtop);
4458   __ pop_ptr(rax); // discard object
4459 }
4460 
4461 // Wide instructions
4462 void TemplateTable::wide() {
4463   transition(vtos, vtos);
4464   __ load_unsigned_byte(rbx, at_bcp(1));
4465   ExternalAddress wtable((address)Interpreter::_wentry_point);
4466   __ jump(ArrayAddress(wtable, Address(noreg, rbx, Address::times_ptr)), rscratch1);
4467   // Note: the rbcp increment step is part of the individual wide bytecode implementations
4468 }
4469 
4470 // Multi arrays
4471 void TemplateTable::multianewarray() {
4472   transition(vtos, atos);
4473 
4474   Register rarg = LP64_ONLY(c_rarg1) NOT_LP64(rax);
4475   __ load_unsigned_byte(rax, at_bcp(3)); // get number of dimensions
4476   // last dim is on top of stack; we want address of first one:
4477   // first_addr = last_addr + (ndims - 1) * stackElementSize - 1*wordsize
4478   // the latter wordSize to point to the beginning of the array.
4479   __ lea(rarg, Address(rsp, rax, Interpreter::stackElementScale(), -wordSize));
4480   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray), rarg);
4481   __ load_unsigned_byte(rbx, at_bcp(3));
4482   __ lea(rsp, Address(rsp, rbx, Interpreter::stackElementScale()));  // get rid of counts
4483 }