1 /*
   2  * Copyright (c) 2013, Red Hat Inc.
   3  * Copyright (c) 2003, 2011, Oracle and/or its affiliates.
   4  * All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "interpreter/interpreter.hpp"
  30 #include "interpreter/interpreterRuntime.hpp"
  31 #include "interpreter/templateTable.hpp"
  32 #include "memory/universe.inline.hpp"
  33 #include "oops/methodData.hpp"
  34 #include "oops/method.hpp"
  35 #include "oops/objArrayKlass.hpp"
  36 #include "oops/oop.inline.hpp"
  37 #include "prims/methodHandles.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/synchronizer.hpp"
  41 #if INCLUDE_ALL_GCS
  42 #include "shenandoahBarrierSetAssembler_aarch64.hpp"
  43 #endif
  44 
  45 #ifndef CC_INTERP
  46 
  47 #define __ _masm->
  48 
  49 // Platform-dependent initialization
  50 
  51 void TemplateTable::pd_initialize() {
  52   // No amd64 specific initialization
  53 }
  54 
  55 // Address computation: local variables
  56 
  57 static inline Address iaddress(int n) {
  58   return Address(rlocals, Interpreter::local_offset_in_bytes(n));
  59 }
  60 
  61 static inline Address laddress(int n) {
  62   return iaddress(n + 1);
  63 }
  64 
  65 static inline Address faddress(int n) {
  66   return iaddress(n);
  67 }
  68 
  69 static inline Address daddress(int n) {
  70   return laddress(n);
  71 }
  72 
  73 static inline Address aaddress(int n) {
  74   return iaddress(n);
  75 }
  76 
  77 static inline Address iaddress(Register r) {
  78   return Address(rlocals, r, Address::lsl(3));
  79 }
  80 
  81 static inline Address laddress(Register r, Register scratch,
  82                                InterpreterMacroAssembler* _masm) {
  83   __ lea(scratch, Address(rlocals, r, Address::lsl(3)));
  84   return Address(scratch, Interpreter::local_offset_in_bytes(1));
  85 }
  86 
  87 static inline Address faddress(Register r) {
  88   return iaddress(r);
  89 }
  90 
  91 static inline Address daddress(Register r, Register scratch,
  92                                InterpreterMacroAssembler* _masm) {
  93   return laddress(r, scratch, _masm);
  94 }
  95 
  96 static inline Address aaddress(Register r) {
  97   return iaddress(r);
  98 }
  99 
 100 static inline Address at_rsp() {
 101   return Address(esp, 0);
 102 }
 103 
 104 // At top of Java expression stack which may be different than esp().  It
 105 // isn't for category 1 objects.
 106 static inline Address at_tos   () {
 107   return Address(esp,  Interpreter::expr_offset_in_bytes(0));
 108 }
 109 
 110 static inline Address at_tos_p1() {
 111   return Address(esp,  Interpreter::expr_offset_in_bytes(1));
 112 }
 113 
 114 static inline Address at_tos_p2() {
 115   return Address(esp,  Interpreter::expr_offset_in_bytes(2));
 116 }
 117 
 118 static inline Address at_tos_p3() {
 119   return Address(esp,  Interpreter::expr_offset_in_bytes(3));
 120 }
 121 
 122 static inline Address at_tos_p4() {
 123   return Address(esp,  Interpreter::expr_offset_in_bytes(4));
 124 }
 125 
 126 static inline Address at_tos_p5() {
 127   return Address(esp,  Interpreter::expr_offset_in_bytes(5));
 128 }
 129 
 130 // Condition conversion
 131 static Assembler::Condition j_not(TemplateTable::Condition cc) {
 132   switch (cc) {
 133   case TemplateTable::equal        : return Assembler::NE;
 134   case TemplateTable::not_equal    : return Assembler::EQ;
 135   case TemplateTable::less         : return Assembler::GE;
 136   case TemplateTable::less_equal   : return Assembler::GT;
 137   case TemplateTable::greater      : return Assembler::LE;
 138   case TemplateTable::greater_equal: return Assembler::LT;
 139   }
 140   ShouldNotReachHere();
 141   return Assembler::EQ;
 142 }
 143 
 144 
 145 // Miscelaneous helper routines
 146 // Store an oop (or NULL) at the Address described by obj.
 147 // If val == noreg this means store a NULL
 148 static void do_oop_store(InterpreterMacroAssembler* _masm,
 149                          Address obj,
 150                          Register val,
 151                          BarrierSet::Name barrier,
 152                          bool precise) {
 153   assert(val == noreg || val == r0, "parameter is just for looks");
 154   switch (barrier) {
 155 #if INCLUDE_ALL_GCS
 156     case BarrierSet::G1SATBCT:
 157     case BarrierSet::G1SATBCTLogging:
 158       {
 159         // flatten object address if needed
 160         if (obj.index() == noreg && obj.offset() == 0) {
 161           if (obj.base() != r3) {
 162             __ mov(r3, obj.base());
 163           }
 164         } else {
 165           __ lea(r3, obj);
 166         }
 167         __ g1_write_barrier_pre(r3 /* obj */,
 168                                 r1 /* pre_val */,
 169                                 rthread /* thread */,
 170                                 r10  /* tmp */,
 171                                 val != noreg /* tosca_live */,
 172                                 false /* expand_call */);
 173         if (val == noreg) {
 174           __ store_heap_oop_null(Address(r3, 0));
 175         } else {
 176           // G1 barrier needs uncompressed oop for region cross check.
 177           Register new_val = val;
 178           if (UseCompressedOops) {
 179             new_val = rscratch2;
 180             __ mov(new_val, val);
 181           }
 182           __ store_heap_oop(Address(r3, 0), val);
 183           __ g1_write_barrier_post(r3 /* store_adr */,
 184                                    new_val /* new_val */,
 185                                    rthread /* thread */,
 186                                    r10 /* tmp */,
 187                                    r1 /* tmp2 */);
 188         }
 189 
 190       }
 191       break;
 192     case BarrierSet::ShenandoahBarrierSet:
 193       {
 194         // flatten object address if needed
 195         if (obj.index() == noreg && obj.offset() == 0) {
 196           if (obj.base() != r3) {
 197             __ mov(r3, obj.base());
 198           }
 199         } else {
 200           __ lea(r3, obj);
 201         }
 202         if (ShenandoahSATBBarrier) {
 203           __ g1_write_barrier_pre(r3 /* obj */,
 204                                   r1 /* pre_val */,
 205                                   rthread /* thread */,
 206                                   r10  /* tmp */,
 207                                   val != noreg /* tosca_live */,
 208                                   false /* expand_call */);
 209         }
 210         if (val == noreg) {
 211           __ store_heap_oop_null(Address(r3, 0));
 212         } else {
 213           if (ShenandoahStoreValEnqueueBarrier) {
 214             ShenandoahBarrierSetAssembler::bsasm()->storeval_barrier(_masm, val, r10);
 215           }
 216           __ store_heap_oop(Address(r3, 0), val);
 217         }
 218 
 219       }
 220       break;
 221 #endif // INCLUDE_ALL_GCS
 222     case BarrierSet::CardTableModRef:
 223     case BarrierSet::CardTableExtension:
 224       {
 225         if (val == noreg) {
 226           __ store_heap_oop_null(obj);
 227         } else {
 228           __ store_heap_oop(obj, val);
 229           // flatten object address if needed
 230           if (!precise || (obj.index() == noreg && obj.offset() == 0)) {
 231             __ store_check(obj.base());
 232           } else {
 233             __ lea(r3, obj);
 234             __ store_check(r3);
 235           }
 236         }
 237       }
 238       break;
 239     case BarrierSet::ModRef:
 240     case BarrierSet::Other:
 241       if (val == noreg) {
 242         __ store_heap_oop_null(obj);
 243       } else {
 244         __ store_heap_oop(obj, val);
 245       }
 246       break;
 247     default      :
 248       ShouldNotReachHere();
 249 
 250   }
 251 }
 252 
 253 Address TemplateTable::at_bcp(int offset) {
 254   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 255   return Address(rbcp, offset);
 256 }
 257 
 258 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 259                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 260                                    int byte_no)
 261 {
 262   if (!RewriteBytecodes)  return;
 263   Label L_patch_done;
 264 
 265   switch (bc) {
 266   case Bytecodes::_fast_aputfield:
 267   case Bytecodes::_fast_bputfield:
 268   case Bytecodes::_fast_zputfield:
 269   case Bytecodes::_fast_cputfield:
 270   case Bytecodes::_fast_dputfield:
 271   case Bytecodes::_fast_fputfield:
 272   case Bytecodes::_fast_iputfield:
 273   case Bytecodes::_fast_lputfield:
 274   case Bytecodes::_fast_sputfield:
 275     {
 276       // We skip bytecode quickening for putfield instructions when
 277       // the put_code written to the constant pool cache is zero.
 278       // This is required so that every execution of this instruction
 279       // calls out to InterpreterRuntime::resolve_get_put to do
 280       // additional, required work.
 281       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 282       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 283       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 284       __ movw(bc_reg, bc);
 285       __ cmpw(temp_reg, (unsigned) 0);
 286       __ br(Assembler::EQ, L_patch_done);  // don't patch
 287     }
 288     break;
 289   default:
 290     assert(byte_no == -1, "sanity");
 291     // the pair bytecodes have already done the load.
 292     if (load_bc_into_bc_reg) {
 293       __ movw(bc_reg, bc);
 294     }
 295   }
 296 
 297   if (JvmtiExport::can_post_breakpoint()) {
 298     Label L_fast_patch;
 299     // if a breakpoint is present we can't rewrite the stream directly
 300     __ load_unsigned_byte(temp_reg, at_bcp(0));
 301     __ cmpw(temp_reg, Bytecodes::_breakpoint);
 302     __ br(Assembler::NE, L_fast_patch);
 303     // Let breakpoint table handling rewrite to quicker bytecode
 304     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), rmethod, rbcp, bc_reg);
 305     __ b(L_patch_done);
 306     __ bind(L_fast_patch);
 307   }
 308 
 309 #ifdef ASSERT
 310   Label L_okay;
 311   __ load_unsigned_byte(temp_reg, at_bcp(0));
 312   __ cmpw(temp_reg, (int) Bytecodes::java_code(bc));
 313   __ br(Assembler::EQ, L_okay);
 314   __ cmpw(temp_reg, bc_reg);
 315   __ br(Assembler::EQ, L_okay);
 316   __ stop("patching the wrong bytecode");
 317   __ bind(L_okay);
 318 #endif
 319 
 320   // patch bytecode
 321   __ strb(bc_reg, at_bcp(0));
 322   __ bind(L_patch_done);
 323 }
 324 
 325 
 326 // Individual instructions
 327 
 328 void TemplateTable::nop() {
 329   transition(vtos, vtos);
 330   // nothing to do
 331 }
 332 
 333 void TemplateTable::shouldnotreachhere() {
 334   transition(vtos, vtos);
 335   __ stop("shouldnotreachhere bytecode");
 336 }
 337 
 338 void TemplateTable::aconst_null()
 339 {
 340   transition(vtos, atos);
 341   __ mov(r0, 0);
 342 }
 343 
 344 void TemplateTable::iconst(int value)
 345 {
 346   transition(vtos, itos);
 347   __ mov(r0, value);
 348 }
 349 
 350 void TemplateTable::lconst(int value)
 351 {
 352   __ mov(r0, value);
 353 }
 354 
 355 void TemplateTable::fconst(int value)
 356 {
 357   transition(vtos, ftos);
 358   switch (value) {
 359   case 0:
 360     __ fmovs(v0, zr);
 361     break;
 362   case 1:
 363     __ fmovs(v0, 1.0);
 364     break;
 365   case 2:
 366     __ fmovs(v0, 2.0);
 367     break;
 368   default:
 369     ShouldNotReachHere();
 370     break;
 371   }
 372 }
 373 
 374 void TemplateTable::dconst(int value)
 375 {
 376   transition(vtos, dtos);
 377   switch (value) {
 378   case 0:
 379     __ fmovd(v0, zr);
 380     break;
 381   case 1:
 382     __ fmovd(v0, 1.0);
 383     break;
 384   case 2:
 385     __ fmovd(v0, 2.0);
 386     break;
 387   default:
 388     ShouldNotReachHere();
 389     break;
 390   }
 391 }
 392 
 393 void TemplateTable::bipush()
 394 {
 395   transition(vtos, itos);
 396   __ load_signed_byte32(r0, at_bcp(1));
 397 }
 398 
 399 void TemplateTable::sipush()
 400 {
 401   transition(vtos, itos);
 402   __ load_unsigned_short(r0, at_bcp(1));
 403   __ revw(r0, r0);
 404   __ asrw(r0, r0, 16);
 405 }
 406 
 407 void TemplateTable::ldc(bool wide)
 408 {
 409   transition(vtos, vtos);
 410   Label call_ldc, notFloat, notClass, Done;
 411 
 412   if (wide) {
 413     __ get_unsigned_2_byte_index_at_bcp(r1, 1);
 414   } else {
 415     __ load_unsigned_byte(r1, at_bcp(1));
 416   }
 417   __ get_cpool_and_tags(r2, r0);
 418 
 419   const int base_offset = ConstantPool::header_size() * wordSize;
 420   const int tags_offset = Array<u1>::base_offset_in_bytes();
 421 
 422   // get type
 423   __ add(r3, r1, tags_offset);
 424   __ lea(r3, Address(r0, r3));
 425   __ ldarb(r3, r3);
 426 
 427   // unresolved class - get the resolved class
 428   __ cmp(r3, JVM_CONSTANT_UnresolvedClass);
 429   __ br(Assembler::EQ, call_ldc);
 430 
 431   // unresolved class in error state - call into runtime to throw the error
 432   // from the first resolution attempt
 433   __ cmp(r3, JVM_CONSTANT_UnresolvedClassInError);
 434   __ br(Assembler::EQ, call_ldc);
 435 
 436   // resolved class - need to call vm to get java mirror of the class
 437   __ cmp(r3, JVM_CONSTANT_Class);
 438   __ br(Assembler::NE, notClass);
 439 
 440   __ bind(call_ldc);
 441   __ mov(c_rarg1, wide);
 442   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 443   __ push_ptr(r0);
 444   __ verify_oop(r0);
 445   __ b(Done);
 446 
 447   __ bind(notClass);
 448   __ cmp(r3, JVM_CONSTANT_Float);
 449   __ br(Assembler::NE, notFloat);
 450   // ftos
 451   __ adds(r1, r2, r1, Assembler::LSL, 3);
 452   __ ldrs(v0, Address(r1, base_offset));
 453   __ push_f();
 454   __ b(Done);
 455 
 456   __ bind(notFloat);
 457 #ifdef ASSERT
 458   {
 459     Label L;
 460     __ cmp(r3, JVM_CONSTANT_Integer);
 461     __ br(Assembler::EQ, L);
 462     // String and Object are rewritten to fast_aldc
 463     __ stop("unexpected tag type in ldc");
 464     __ bind(L);
 465   }
 466 #endif
 467   // itos JVM_CONSTANT_Integer only
 468   __ adds(r1, r2, r1, Assembler::LSL, 3);
 469   __ ldrw(r0, Address(r1, base_offset));
 470   __ push_i(r0);
 471   __ bind(Done);
 472 }
 473 
 474 // Fast path for caching oop constants.
 475 void TemplateTable::fast_aldc(bool wide)
 476 {
 477   transition(vtos, atos);
 478 
 479   Register result = r0;
 480   Register tmp = r1;
 481   int index_size = wide ? sizeof(u2) : sizeof(u1);
 482 
 483   Label resolved;
 484 
 485   // We are resolved if the resolved reference cache entry contains a
 486   // non-null object (String, MethodType, etc.)
 487   assert_different_registers(result, tmp);
 488   __ get_cache_index_at_bcp(tmp, 1, index_size);
 489   __ load_resolved_reference_at_index(result, tmp);
 490   __ cbnz(result, resolved);
 491 
 492   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 493 
 494   // first time invocation - must resolve first
 495   __ mov(tmp, (int)bytecode());
 496   __ call_VM(result, entry, tmp);
 497 
 498   __ bind(resolved);
 499 
 500   if (VerifyOops) {
 501     __ verify_oop(result);
 502   }
 503 }
 504 
 505 void TemplateTable::ldc2_w()
 506 {
 507   transition(vtos, vtos);
 508   Label Long, Done;
 509   __ get_unsigned_2_byte_index_at_bcp(r0, 1);
 510 
 511   __ get_cpool_and_tags(r1, r2);
 512   const int base_offset = ConstantPool::header_size() * wordSize;
 513   const int tags_offset = Array<u1>::base_offset_in_bytes();
 514 
 515   // get type
 516   __ lea(r2, Address(r2, r0, Address::lsl(0)));
 517   __ load_unsigned_byte(r2, Address(r2, tags_offset));
 518   __ cmpw(r2, (int)JVM_CONSTANT_Double);
 519   __ br(Assembler::NE, Long);
 520   // dtos
 521   __ lea (r2, Address(r1, r0, Address::lsl(3)));
 522   __ ldrd(v0, Address(r2, base_offset));
 523   __ push_d();
 524   __ b(Done);
 525 
 526   __ bind(Long);
 527   // ltos
 528   __ lea(r0, Address(r1, r0, Address::lsl(3)));
 529   __ ldr(r0, Address(r0, base_offset));
 530   __ push_l();
 531 
 532   __ bind(Done);
 533 }
 534 
 535 void TemplateTable::locals_index(Register reg, int offset)
 536 {
 537   __ ldrb(reg, at_bcp(offset));
 538   __ neg(reg, reg);
 539 }
 540 
 541 void TemplateTable::iload()
 542 {
 543   transition(vtos, itos);
 544   if (RewriteFrequentPairs) {
 545     Label rewrite, done;
 546     Register bc = r4;
 547 
 548     // get next bytecode
 549     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 550 
 551     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 552     // last two iloads in a pair.  Comparing against fast_iload means that
 553     // the next bytecode is neither an iload or a caload, and therefore
 554     // an iload pair.
 555     __ cmpw(r1, Bytecodes::_iload);
 556     __ br(Assembler::EQ, done);
 557 
 558     // if _fast_iload rewrite to _fast_iload2
 559     __ cmpw(r1, Bytecodes::_fast_iload);
 560     __ movw(bc, Bytecodes::_fast_iload2);
 561     __ br(Assembler::EQ, rewrite);
 562 
 563     // if _caload rewrite to _fast_icaload
 564     __ cmpw(r1, Bytecodes::_caload);
 565     __ movw(bc, Bytecodes::_fast_icaload);
 566     __ br(Assembler::EQ, rewrite);
 567 
 568     // else rewrite to _fast_iload
 569     __ movw(bc, Bytecodes::_fast_iload);
 570 
 571     // rewrite
 572     // bc: new bytecode
 573     __ bind(rewrite);
 574     patch_bytecode(Bytecodes::_iload, bc, r1, false);
 575     __ bind(done);
 576 
 577   }
 578 
 579   // do iload, get the local value into tos
 580   locals_index(r1);
 581   __ ldr(r0, iaddress(r1));
 582 
 583 }
 584 
 585 void TemplateTable::fast_iload2()
 586 {
 587   transition(vtos, itos);
 588   locals_index(r1);
 589   __ ldr(r0, iaddress(r1));
 590   __ push(itos);
 591   locals_index(r1, 3);
 592   __ ldr(r0, iaddress(r1));
 593 }
 594 
 595 void TemplateTable::fast_iload()
 596 {
 597   transition(vtos, itos);
 598   locals_index(r1);
 599   __ ldr(r0, iaddress(r1));
 600 }
 601 
 602 void TemplateTable::lload()
 603 {
 604   transition(vtos, ltos);
 605   __ ldrb(r1, at_bcp(1));
 606   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 607   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 608 }
 609 
 610 void TemplateTable::fload()
 611 {
 612   transition(vtos, ftos);
 613   locals_index(r1);
 614   // n.b. we use ldrd here because this is a 64 bit slot
 615   // this is comparable to the iload case
 616   __ ldrd(v0, faddress(r1));
 617 }
 618 
 619 void TemplateTable::dload()
 620 {
 621   transition(vtos, dtos);
 622   __ ldrb(r1, at_bcp(1));
 623   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 624   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 625 }
 626 
 627 void TemplateTable::aload()
 628 {
 629   transition(vtos, atos);
 630   locals_index(r1);
 631   __ ldr(r0, iaddress(r1));
 632 }
 633 
 634 void TemplateTable::locals_index_wide(Register reg) {
 635   __ ldrh(reg, at_bcp(2));
 636   __ rev16w(reg, reg);
 637   __ neg(reg, reg);
 638 }
 639 
 640 void TemplateTable::wide_iload() {
 641   transition(vtos, itos);
 642   locals_index_wide(r1);
 643   __ ldr(r0, iaddress(r1));
 644 }
 645 
 646 void TemplateTable::wide_lload()
 647 {
 648   transition(vtos, ltos);
 649   __ ldrh(r1, at_bcp(2));
 650   __ rev16w(r1, r1);
 651   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 652   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 653 }
 654 
 655 void TemplateTable::wide_fload()
 656 {
 657   transition(vtos, ftos);
 658   locals_index_wide(r1);
 659   // n.b. we use ldrd here because this is a 64 bit slot
 660   // this is comparable to the iload case
 661   __ ldrd(v0, faddress(r1));
 662 }
 663 
 664 void TemplateTable::wide_dload()
 665 {
 666   transition(vtos, dtos);
 667   __ ldrh(r1, at_bcp(2));
 668   __ rev16w(r1, r1);
 669   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 670   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 671 }
 672 
 673 void TemplateTable::wide_aload()
 674 {
 675   transition(vtos, atos);
 676   locals_index_wide(r1);
 677   __ ldr(r0, aaddress(r1));
 678 }
 679 
 680 void TemplateTable::index_check(Register array, Register index)
 681 {
 682   // destroys r1, rscratch1
 683   // check array
 684   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
 685   // sign extend index for use by indexed load
 686   // __ movl2ptr(index, index);
 687   // check index
 688   Register length = rscratch1;
 689   __ ldrw(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 690   __ cmpw(index, length);
 691   if (index != r1) {
 692     // ??? convention: move aberrant index into r1 for exception message
 693     assert(r1 != array, "different registers");
 694     __ mov(r1, index);
 695   }
 696   Label ok;
 697   __ br(Assembler::LO, ok);
 698   __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 699   __ br(rscratch1);
 700   __ bind(ok);
 701 }
 702 
 703 void TemplateTable::iaload()
 704 {
 705   transition(itos, itos);
 706   __ mov(r1, r0);
 707   __ pop_ptr(r0);
 708   // r0: array
 709   // r1: index
 710   index_check(r0, r1); // leaves index in r1, kills rscratch1
 711   __ lea(r1, Address(r0, r1, Address::uxtw(2)));
 712   __ ldrw(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_INT)));
 713 }
 714 
 715 void TemplateTable::laload()
 716 {
 717   transition(itos, ltos);
 718   __ mov(r1, r0);
 719   __ pop_ptr(r0);
 720   // r0: array
 721   // r1: index
 722   index_check(r0, r1); // leaves index in r1, kills rscratch1
 723   __ lea(r1, Address(r0, r1, Address::uxtw(3)));
 724   __ ldr(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_LONG)));
 725 }
 726 
 727 void TemplateTable::faload()
 728 {
 729   transition(itos, ftos);
 730   __ mov(r1, r0);
 731   __ pop_ptr(r0);
 732   // r0: array
 733   // r1: index
 734   index_check(r0, r1); // leaves index in r1, kills rscratch1
 735   __ lea(r1,  Address(r0, r1, Address::uxtw(2)));
 736   __ ldrs(v0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
 737 }
 738 
 739 void TemplateTable::daload()
 740 {
 741   transition(itos, dtos);
 742   __ mov(r1, r0);
 743   __ pop_ptr(r0);
 744   // r0: array
 745   // r1: index
 746   index_check(r0, r1); // leaves index in r1, kills rscratch1
 747   __ lea(r1,  Address(r0, r1, Address::uxtw(3)));
 748   __ ldrd(v0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
 749 }
 750 
 751 void TemplateTable::aaload()
 752 {
 753   transition(itos, atos);
 754   __ mov(r1, r0);
 755   __ pop_ptr(r0);
 756   // r0: array
 757   // r1: index
 758   index_check(r0, r1); // leaves index in r1, kills rscratch1
 759   int s = (UseCompressedOops ? 2 : 3);
 760   __ lea(r1, Address(r0, r1, Address::uxtw(s)));
 761   __ load_heap_oop(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
 762 }
 763 
 764 void TemplateTable::baload()
 765 {
 766   transition(itos, itos);
 767   __ mov(r1, r0);
 768   __ pop_ptr(r0);
 769   // r0: array
 770   // r1: index
 771   index_check(r0, r1); // leaves index in r1, kills rscratch1
 772   __ lea(r1,  Address(r0, r1, Address::uxtw(0)));
 773   __ load_signed_byte(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_BYTE)));
 774 }
 775 
 776 void TemplateTable::caload()
 777 {
 778   transition(itos, itos);
 779   __ mov(r1, r0);
 780   __ pop_ptr(r0);
 781   // r0: array
 782   // r1: index
 783   index_check(r0, r1); // leaves index in r1, kills rscratch1
 784   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 785   __ load_unsigned_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 786 }
 787 
 788 // iload followed by caload frequent pair
 789 void TemplateTable::fast_icaload()
 790 {
 791   transition(vtos, itos);
 792   // load index out of locals
 793   locals_index(r2);
 794   __ ldr(r1, iaddress(r2));
 795 
 796   __ pop_ptr(r0);
 797 
 798   // r0: array
 799   // r1: index
 800   index_check(r0, r1); // leaves index in r1, kills rscratch1
 801   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 802   __ load_unsigned_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 803 }
 804 
 805 void TemplateTable::saload()
 806 {
 807   transition(itos, itos);
 808   __ mov(r1, r0);
 809   __ pop_ptr(r0);
 810   // r0: array
 811   // r1: index
 812   index_check(r0, r1); // leaves index in r1, kills rscratch1
 813   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 814   __ load_signed_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_SHORT)));
 815 }
 816 
 817 void TemplateTable::iload(int n)
 818 {
 819   transition(vtos, itos);
 820   __ ldr(r0, iaddress(n));
 821 }
 822 
 823 void TemplateTable::lload(int n)
 824 {
 825   transition(vtos, ltos);
 826   __ ldr(r0, laddress(n));
 827 }
 828 
 829 void TemplateTable::fload(int n)
 830 {
 831   transition(vtos, ftos);
 832   __ ldrs(v0, faddress(n));
 833 }
 834 
 835 void TemplateTable::dload(int n)
 836 {
 837   transition(vtos, dtos);
 838   __ ldrd(v0, daddress(n));
 839 }
 840 
 841 void TemplateTable::aload(int n)
 842 {
 843   transition(vtos, atos);
 844   __ ldr(r0, iaddress(n));
 845 }
 846 
 847 void TemplateTable::aload_0()
 848 {
 849   // According to bytecode histograms, the pairs:
 850   //
 851   // _aload_0, _fast_igetfield
 852   // _aload_0, _fast_agetfield
 853   // _aload_0, _fast_fgetfield
 854   //
 855   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 856   // _aload_0 bytecode checks if the next bytecode is either
 857   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 858   // rewrites the current bytecode into a pair bytecode; otherwise it
 859   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 860   // the pair check anymore.
 861   //
 862   // Note: If the next bytecode is _getfield, the rewrite must be
 863   //       delayed, otherwise we may miss an opportunity for a pair.
 864   //
 865   // Also rewrite frequent pairs
 866   //   aload_0, aload_1
 867   //   aload_0, iload_1
 868   // These bytecodes with a small amount of code are most profitable
 869   // to rewrite
 870   if (RewriteFrequentPairs) {
 871     Label rewrite, done;
 872     const Register bc = r4;
 873 
 874     // get next bytecode
 875     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 876 
 877     // do actual aload_0
 878     aload(0);
 879 
 880     // if _getfield then wait with rewrite
 881     __ cmpw(r1, Bytecodes::Bytecodes::_getfield);
 882     __ br(Assembler::EQ, done);
 883 
 884     // if _igetfield then reqrite to _fast_iaccess_0
 885     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 886     __ cmpw(r1, Bytecodes::_fast_igetfield);
 887     __ movw(bc, Bytecodes::_fast_iaccess_0);
 888     __ br(Assembler::EQ, rewrite);
 889 
 890     // if _agetfield then reqrite to _fast_aaccess_0
 891     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 892     __ cmpw(r1, Bytecodes::_fast_agetfield);
 893     __ movw(bc, Bytecodes::_fast_aaccess_0);
 894     __ br(Assembler::EQ, rewrite);
 895 
 896     // if _fgetfield then reqrite to _fast_faccess_0
 897     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 898     __ cmpw(r1, Bytecodes::_fast_fgetfield);
 899     __ movw(bc, Bytecodes::_fast_faccess_0);
 900     __ br(Assembler::EQ, rewrite);
 901 
 902     // else rewrite to _fast_aload0
 903     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 904     __ movw(bc, Bytecodes::Bytecodes::_fast_aload_0);
 905 
 906     // rewrite
 907     // bc: new bytecode
 908     __ bind(rewrite);
 909     patch_bytecode(Bytecodes::_aload_0, bc, r1, false);
 910 
 911     __ bind(done);
 912   } else {
 913     aload(0);
 914   }
 915 }
 916 
 917 void TemplateTable::istore()
 918 {
 919   transition(itos, vtos);
 920   locals_index(r1);
 921   // FIXME: We're being very pernickerty here storing a jint in a
 922   // local with strw, which costs an extra instruction over what we'd
 923   // be able to do with a simple str.  We should just store the whole
 924   // word.
 925   __ lea(rscratch1, iaddress(r1));
 926   __ strw(r0, Address(rscratch1));
 927 }
 928 
 929 void TemplateTable::lstore()
 930 {
 931   transition(ltos, vtos);
 932   locals_index(r1);
 933   __ str(r0, laddress(r1, rscratch1, _masm));
 934 }
 935 
 936 void TemplateTable::fstore() {
 937   transition(ftos, vtos);
 938   locals_index(r1);
 939   __ lea(rscratch1, iaddress(r1));
 940   __ strs(v0, Address(rscratch1));
 941 }
 942 
 943 void TemplateTable::dstore() {
 944   transition(dtos, vtos);
 945   locals_index(r1);
 946   __ strd(v0, daddress(r1, rscratch1, _masm));
 947 }
 948 
 949 void TemplateTable::astore()
 950 {
 951   transition(vtos, vtos);
 952   __ pop_ptr(r0);
 953   locals_index(r1);
 954   __ str(r0, aaddress(r1));
 955 }
 956 
 957 void TemplateTable::wide_istore() {
 958   transition(vtos, vtos);
 959   __ pop_i();
 960   locals_index_wide(r1);
 961   __ lea(rscratch1, iaddress(r1));
 962   __ strw(r0, Address(rscratch1));
 963 }
 964 
 965 void TemplateTable::wide_lstore() {
 966   transition(vtos, vtos);
 967   __ pop_l();
 968   locals_index_wide(r1);
 969   __ str(r0, laddress(r1, rscratch1, _masm));
 970 }
 971 
 972 void TemplateTable::wide_fstore() {
 973   transition(vtos, vtos);
 974   __ pop_f();
 975   locals_index_wide(r1);
 976   __ lea(rscratch1, faddress(r1));
 977   __ strs(v0, rscratch1);
 978 }
 979 
 980 void TemplateTable::wide_dstore() {
 981   transition(vtos, vtos);
 982   __ pop_d();
 983   locals_index_wide(r1);
 984   __ strd(v0, daddress(r1, rscratch1, _masm));
 985 }
 986 
 987 void TemplateTable::wide_astore() {
 988   transition(vtos, vtos);
 989   __ pop_ptr(r0);
 990   locals_index_wide(r1);
 991   __ str(r0, aaddress(r1));
 992 }
 993 
 994 void TemplateTable::iastore() {
 995   transition(itos, vtos);
 996   __ pop_i(r1);
 997   __ pop_ptr(r3);
 998   // r0: value
 999   // r1: index
1000   // r3: array
1001   index_check(r3, r1); // prefer index in r1
1002   __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
1003   __ strw(r0, Address(rscratch1,
1004                       arrayOopDesc::base_offset_in_bytes(T_INT)));
1005 }
1006 
1007 void TemplateTable::lastore() {
1008   transition(ltos, vtos);
1009   __ pop_i(r1);
1010   __ pop_ptr(r3);
1011   // r0: value
1012   // r1: index
1013   // r3: array
1014   index_check(r3, r1); // prefer index in r1
1015   __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
1016   __ str(r0, Address(rscratch1,
1017                       arrayOopDesc::base_offset_in_bytes(T_LONG)));
1018 }
1019 
1020 void TemplateTable::fastore() {
1021   transition(ftos, vtos);
1022   __ pop_i(r1);
1023   __ pop_ptr(r3);
1024   // v0: value
1025   // r1:  index
1026   // r3:  array
1027   index_check(r3, r1); // prefer index in r1
1028   __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
1029   __ strs(v0, Address(rscratch1,
1030                       arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
1031 }
1032 
1033 void TemplateTable::dastore() {
1034   transition(dtos, vtos);
1035   __ pop_i(r1);
1036   __ pop_ptr(r3);
1037   // v0: value
1038   // r1:  index
1039   // r3:  array
1040   index_check(r3, r1); // prefer index in r1
1041   __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
1042   __ strd(v0, Address(rscratch1,
1043                       arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
1044 }
1045 
1046 void TemplateTable::aastore() {
1047   Label is_null, ok_is_subtype, done;
1048   transition(vtos, vtos);
1049   // stack: ..., array, index, value
1050   __ ldr(r0, at_tos());    // value
1051   __ ldr(r2, at_tos_p1()); // index
1052   __ ldr(r3, at_tos_p2()); // array
1053 
1054   Address element_address(r4, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
1055 
1056   index_check(r3, r2);     // kills r1
1057   __ lea(r4, Address(r3, r2, Address::uxtw(UseCompressedOops? 2 : 3)));
1058 
1059   // do array store check - check for NULL value first
1060   __ cbz(r0, is_null);
1061 
1062   // Move subklass into r1
1063   __ load_klass(r1, r0);
1064   // Move superklass into r0
1065   __ load_klass(r0, r3);
1066   __ ldr(r0, Address(r0,
1067                      ObjArrayKlass::element_klass_offset()));
1068   // Compress array + index*oopSize + 12 into a single register.  Frees r2.
1069 
1070   // Generate subtype check.  Blows r2, r5
1071   // Superklass in r0.  Subklass in r1.
1072   __ gen_subtype_check(r1, ok_is_subtype);
1073 
1074   // Come here on failure
1075   // object is at TOS
1076   __ b(Interpreter::_throw_ArrayStoreException_entry);
1077 
1078   // Come here on success
1079   __ bind(ok_is_subtype);
1080 
1081   // Get the value we will store
1082   __ ldr(r0, at_tos());
1083   // Now store using the appropriate barrier
1084   do_oop_store(_masm, element_address, r0, _bs->kind(), true);
1085   __ b(done);
1086 
1087   // Have a NULL in r0, r3=array, r2=index.  Store NULL at ary[idx]
1088   __ bind(is_null);
1089   __ profile_null_seen(r2);
1090 
1091   // Store a NULL
1092   do_oop_store(_masm, element_address, noreg, _bs->kind(), true);
1093 
1094   // Pop stack arguments
1095   __ bind(done);
1096   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1097 }
1098 
1099 void TemplateTable::bastore()
1100 {
1101   transition(itos, vtos);
1102   __ pop_i(r1);
1103   __ pop_ptr(r3);
1104   // r0: value
1105   // r1: index
1106   // r3: array
1107   index_check(r3, r1); // prefer index in r1
1108 
1109   // Need to check whether array is boolean or byte
1110   // since both types share the bastore bytecode.
1111   __ load_klass(r2, r3);
1112   __ ldrw(r2, Address(r2, Klass::layout_helper_offset()));
1113   int diffbit = Klass::layout_helper_boolean_diffbit();
1114   __ andw(rscratch1, r2, diffbit);
1115   Label L_skip;
1116   __ cbzw(rscratch1, L_skip);
1117   __ andw(r0, r0, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1118   __ bind(L_skip);
1119 
1120   __ lea(rscratch1, Address(r3, r1, Address::uxtw(0)));
1121   __ strb(r0, Address(rscratch1,
1122                       arrayOopDesc::base_offset_in_bytes(T_BYTE)));
1123 }
1124 
1125 void TemplateTable::castore()
1126 {
1127   transition(itos, vtos);
1128   __ pop_i(r1);
1129   __ pop_ptr(r3);
1130   // r0: value
1131   // r1: index
1132   // r3: array
1133   index_check(r3, r1); // prefer index in r1
1134   __ lea(rscratch1, Address(r3, r1, Address::uxtw(1)));
1135   __ strh(r0, Address(rscratch1,
1136                       arrayOopDesc::base_offset_in_bytes(T_CHAR)));
1137 }
1138 
1139 void TemplateTable::sastore()
1140 {
1141   castore();
1142 }
1143 
1144 void TemplateTable::istore(int n)
1145 {
1146   transition(itos, vtos);
1147   __ str(r0, iaddress(n));
1148 }
1149 
1150 void TemplateTable::lstore(int n)
1151 {
1152   transition(ltos, vtos);
1153   __ str(r0, laddress(n));
1154 }
1155 
1156 void TemplateTable::fstore(int n)
1157 {
1158   transition(ftos, vtos);
1159   __ strs(v0, faddress(n));
1160 }
1161 
1162 void TemplateTable::dstore(int n)
1163 {
1164   transition(dtos, vtos);
1165   __ strd(v0, daddress(n));
1166 }
1167 
1168 void TemplateTable::astore(int n)
1169 {
1170   transition(vtos, vtos);
1171   __ pop_ptr(r0);
1172   __ str(r0, iaddress(n));
1173 }
1174 
1175 void TemplateTable::pop()
1176 {
1177   transition(vtos, vtos);
1178   __ add(esp, esp, Interpreter::stackElementSize);
1179 }
1180 
1181 void TemplateTable::pop2()
1182 {
1183   transition(vtos, vtos);
1184   __ add(esp, esp, 2 * Interpreter::stackElementSize);
1185 }
1186 
1187 void TemplateTable::dup()
1188 {
1189   transition(vtos, vtos);
1190   __ ldr(r0, Address(esp, 0));
1191   __ push(r0);
1192   // stack: ..., a, a
1193 }
1194 
1195 void TemplateTable::dup_x1()
1196 {
1197   transition(vtos, vtos);
1198   // stack: ..., a, b
1199   __ ldr(r0, at_tos());  // load b
1200   __ ldr(r2, at_tos_p1());  // load a
1201   __ str(r0, at_tos_p1());  // store b
1202   __ str(r2, at_tos());  // store a
1203   __ push(r0);                  // push b
1204   // stack: ..., b, a, b
1205 }
1206 
1207 void TemplateTable::dup_x2()
1208 {
1209   transition(vtos, vtos);
1210   // stack: ..., a, b, c
1211   __ ldr(r0, at_tos());  // load c
1212   __ ldr(r2, at_tos_p2());  // load a
1213   __ str(r0, at_tos_p2());  // store c in a
1214   __ push(r0);      // push c
1215   // stack: ..., c, b, c, c
1216   __ ldr(r0, at_tos_p2());  // load b
1217   __ str(r2, at_tos_p2());  // store a in b
1218   // stack: ..., c, a, c, c
1219   __ str(r0, at_tos_p1());  // store b in c
1220   // stack: ..., c, a, b, c
1221 }
1222 
1223 void TemplateTable::dup2()
1224 {
1225   transition(vtos, vtos);
1226   // stack: ..., a, b
1227   __ ldr(r0, at_tos_p1());  // load a
1228   __ push(r0);                  // push a
1229   __ ldr(r0, at_tos_p1());  // load b
1230   __ push(r0);                  // push b
1231   // stack: ..., a, b, a, b
1232 }
1233 
1234 void TemplateTable::dup2_x1()
1235 {
1236   transition(vtos, vtos);
1237   // stack: ..., a, b, c
1238   __ ldr(r2, at_tos());  // load c
1239   __ ldr(r0, at_tos_p1());  // load b
1240   __ push(r0);                  // push b
1241   __ push(r2);                  // push c
1242   // stack: ..., a, b, c, b, c
1243   __ str(r2, at_tos_p3());  // store c in b
1244   // stack: ..., a, c, c, b, c
1245   __ ldr(r2, at_tos_p4());  // load a
1246   __ str(r2, at_tos_p2());  // store a in 2nd c
1247   // stack: ..., a, c, a, b, c
1248   __ str(r0, at_tos_p4());  // store b in a
1249   // stack: ..., b, c, a, b, c
1250 }
1251 
1252 void TemplateTable::dup2_x2()
1253 {
1254   transition(vtos, vtos);
1255   // stack: ..., a, b, c, d
1256   __ ldr(r2, at_tos());  // load d
1257   __ ldr(r0, at_tos_p1());  // load c
1258   __ push(r0)            ;      // push c
1259   __ push(r2);                  // push d
1260   // stack: ..., a, b, c, d, c, d
1261   __ ldr(r0, at_tos_p4());  // load b
1262   __ str(r0, at_tos_p2());  // store b in d
1263   __ str(r2, at_tos_p4());  // store d in b
1264   // stack: ..., a, d, c, b, c, d
1265   __ ldr(r2, at_tos_p5());  // load a
1266   __ ldr(r0, at_tos_p3());  // load c
1267   __ str(r2, at_tos_p3());  // store a in c
1268   __ str(r0, at_tos_p5());  // store c in a
1269   // stack: ..., c, d, a, b, c, d
1270 }
1271 
1272 void TemplateTable::swap()
1273 {
1274   transition(vtos, vtos);
1275   // stack: ..., a, b
1276   __ ldr(r2, at_tos_p1());  // load a
1277   __ ldr(r0, at_tos());  // load b
1278   __ str(r2, at_tos());  // store a in b
1279   __ str(r0, at_tos_p1());  // store b in a
1280   // stack: ..., b, a
1281 }
1282 
1283 void TemplateTable::iop2(Operation op)
1284 {
1285   transition(itos, itos);
1286   // r0 <== r1 op r0
1287   __ pop_i(r1);
1288   switch (op) {
1289   case add  : __ addw(r0, r1, r0); break;
1290   case sub  : __ subw(r0, r1, r0); break;
1291   case mul  : __ mulw(r0, r1, r0); break;
1292   case _and : __ andw(r0, r1, r0); break;
1293   case _or  : __ orrw(r0, r1, r0); break;
1294   case _xor : __ eorw(r0, r1, r0); break;
1295   case shl  : __ lslvw(r0, r1, r0); break;
1296   case shr  : __ asrvw(r0, r1, r0); break;
1297   case ushr : __ lsrvw(r0, r1, r0);break;
1298   default   : ShouldNotReachHere();
1299   }
1300 }
1301 
1302 void TemplateTable::lop2(Operation op)
1303 {
1304   transition(ltos, ltos);
1305   // r0 <== r1 op r0
1306   __ pop_l(r1);
1307   switch (op) {
1308   case add  : __ add(r0, r1, r0); break;
1309   case sub  : __ sub(r0, r1, r0); break;
1310   case mul  : __ mul(r0, r1, r0); break;
1311   case _and : __ andr(r0, r1, r0); break;
1312   case _or  : __ orr(r0, r1, r0); break;
1313   case _xor : __ eor(r0, r1, r0); break;
1314   default   : ShouldNotReachHere();
1315   }
1316 }
1317 
1318 void TemplateTable::idiv()
1319 {
1320   transition(itos, itos);
1321   // explicitly check for div0
1322   Label no_div0;
1323   __ cbnzw(r0, no_div0);
1324   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1325   __ br(rscratch1);
1326   __ bind(no_div0);
1327   __ pop_i(r1);
1328   // r0 <== r1 idiv r0
1329   __ corrected_idivl(r0, r1, r0, /* want_remainder */ false);
1330 }
1331 
1332 void TemplateTable::irem()
1333 {
1334   transition(itos, itos);
1335   // explicitly check for div0
1336   Label no_div0;
1337   __ cbnzw(r0, no_div0);
1338   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1339   __ br(rscratch1);
1340   __ bind(no_div0);
1341   __ pop_i(r1);
1342   // r0 <== r1 irem r0
1343   __ corrected_idivl(r0, r1, r0, /* want_remainder */ true);
1344 }
1345 
1346 void TemplateTable::lmul()
1347 {
1348   transition(ltos, ltos);
1349   __ pop_l(r1);
1350   __ mul(r0, r0, r1);
1351 }
1352 
1353 void TemplateTable::ldiv()
1354 {
1355   transition(ltos, ltos);
1356   // explicitly check for div0
1357   Label no_div0;
1358   __ cbnz(r0, no_div0);
1359   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1360   __ br(rscratch1);
1361   __ bind(no_div0);
1362   __ pop_l(r1);
1363   // r0 <== r1 ldiv r0
1364   __ corrected_idivq(r0, r1, r0, /* want_remainder */ false);
1365 }
1366 
1367 void TemplateTable::lrem()
1368 {
1369   transition(ltos, ltos);
1370   // explicitly check for div0
1371   Label no_div0;
1372   __ cbnz(r0, no_div0);
1373   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1374   __ br(rscratch1);
1375   __ bind(no_div0);
1376   __ pop_l(r1);
1377   // r0 <== r1 lrem r0
1378   __ corrected_idivq(r0, r1, r0, /* want_remainder */ true);
1379 }
1380 
1381 void TemplateTable::lshl()
1382 {
1383   transition(itos, ltos);
1384   // shift count is in r0
1385   __ pop_l(r1);
1386   __ lslv(r0, r1, r0);
1387 }
1388 
1389 void TemplateTable::lshr()
1390 {
1391   transition(itos, ltos);
1392   // shift count is in r0
1393   __ pop_l(r1);
1394   __ asrv(r0, r1, r0);
1395 }
1396 
1397 void TemplateTable::lushr()
1398 {
1399   transition(itos, ltos);
1400   // shift count is in r0
1401   __ pop_l(r1);
1402   __ lsrv(r0, r1, r0);
1403 }
1404 
1405 void TemplateTable::fop2(Operation op)
1406 {
1407   transition(ftos, ftos);
1408   switch (op) {
1409   case add:
1410     // n.b. use ldrd because this is a 64 bit slot
1411     __ pop_f(v1);
1412     __ fadds(v0, v1, v0);
1413     break;
1414   case sub:
1415     __ pop_f(v1);
1416     __ fsubs(v0, v1, v0);
1417     break;
1418   case mul:
1419     __ pop_f(v1);
1420     __ fmuls(v0, v1, v0);
1421     break;
1422   case div:
1423     __ pop_f(v1);
1424     __ fdivs(v0, v1, v0);
1425     break;
1426   case rem:
1427     __ fmovs(v1, v0);
1428     __ pop_f(v0);
1429     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem));
1430     break;
1431   default:
1432     ShouldNotReachHere();
1433     break;
1434   }
1435 }
1436 
1437 void TemplateTable::dop2(Operation op)
1438 {
1439   transition(dtos, dtos);
1440   switch (op) {
1441   case add:
1442     // n.b. use ldrd because this is a 64 bit slot
1443     __ pop_d(v1);
1444     __ faddd(v0, v1, v0);
1445     break;
1446   case sub:
1447     __ pop_d(v1);
1448     __ fsubd(v0, v1, v0);
1449     break;
1450   case mul:
1451     __ pop_d(v1);
1452     __ fmuld(v0, v1, v0);
1453     break;
1454   case div:
1455     __ pop_d(v1);
1456     __ fdivd(v0, v1, v0);
1457     break;
1458   case rem:
1459     __ fmovd(v1, v0);
1460     __ pop_d(v0);
1461     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem));
1462     break;
1463   default:
1464     ShouldNotReachHere();
1465     break;
1466   }
1467 }
1468 
1469 void TemplateTable::ineg()
1470 {
1471   transition(itos, itos);
1472   __ negw(r0, r0);
1473 
1474 }
1475 
1476 void TemplateTable::lneg()
1477 {
1478   transition(ltos, ltos);
1479   __ neg(r0, r0);
1480 }
1481 
1482 void TemplateTable::fneg()
1483 {
1484   transition(ftos, ftos);
1485   __ fnegs(v0, v0);
1486 }
1487 
1488 void TemplateTable::dneg()
1489 {
1490   transition(dtos, dtos);
1491   __ fnegd(v0, v0);
1492 }
1493 
1494 void TemplateTable::iinc()
1495 {
1496   transition(vtos, vtos);
1497   __ load_signed_byte(r1, at_bcp(2)); // get constant
1498   locals_index(r2);
1499   __ ldr(r0, iaddress(r2));
1500   __ addw(r0, r0, r1);
1501   __ str(r0, iaddress(r2));
1502 }
1503 
1504 void TemplateTable::wide_iinc()
1505 {
1506   transition(vtos, vtos);
1507   // __ mov(r1, zr);
1508   __ ldrw(r1, at_bcp(2)); // get constant and index
1509   __ rev16(r1, r1);
1510   __ ubfx(r2, r1, 0, 16);
1511   __ neg(r2, r2);
1512   __ sbfx(r1, r1, 16, 16);
1513   __ ldr(r0, iaddress(r2));
1514   __ addw(r0, r0, r1);
1515   __ str(r0, iaddress(r2));
1516 }
1517 
1518 void TemplateTable::convert()
1519 {
1520   // Checking
1521 #ifdef ASSERT
1522   {
1523     TosState tos_in  = ilgl;
1524     TosState tos_out = ilgl;
1525     switch (bytecode()) {
1526     case Bytecodes::_i2l: // fall through
1527     case Bytecodes::_i2f: // fall through
1528     case Bytecodes::_i2d: // fall through
1529     case Bytecodes::_i2b: // fall through
1530     case Bytecodes::_i2c: // fall through
1531     case Bytecodes::_i2s: tos_in = itos; break;
1532     case Bytecodes::_l2i: // fall through
1533     case Bytecodes::_l2f: // fall through
1534     case Bytecodes::_l2d: tos_in = ltos; break;
1535     case Bytecodes::_f2i: // fall through
1536     case Bytecodes::_f2l: // fall through
1537     case Bytecodes::_f2d: tos_in = ftos; break;
1538     case Bytecodes::_d2i: // fall through
1539     case Bytecodes::_d2l: // fall through
1540     case Bytecodes::_d2f: tos_in = dtos; break;
1541     default             : ShouldNotReachHere();
1542     }
1543     switch (bytecode()) {
1544     case Bytecodes::_l2i: // fall through
1545     case Bytecodes::_f2i: // fall through
1546     case Bytecodes::_d2i: // fall through
1547     case Bytecodes::_i2b: // fall through
1548     case Bytecodes::_i2c: // fall through
1549     case Bytecodes::_i2s: tos_out = itos; break;
1550     case Bytecodes::_i2l: // fall through
1551     case Bytecodes::_f2l: // fall through
1552     case Bytecodes::_d2l: tos_out = ltos; break;
1553     case Bytecodes::_i2f: // fall through
1554     case Bytecodes::_l2f: // fall through
1555     case Bytecodes::_d2f: tos_out = ftos; break;
1556     case Bytecodes::_i2d: // fall through
1557     case Bytecodes::_l2d: // fall through
1558     case Bytecodes::_f2d: tos_out = dtos; break;
1559     default             : ShouldNotReachHere();
1560     }
1561     transition(tos_in, tos_out);
1562   }
1563 #endif // ASSERT
1564   // static const int64_t is_nan = 0x8000000000000000L;
1565 
1566   // Conversion
1567   switch (bytecode()) {
1568   case Bytecodes::_i2l:
1569     __ sxtw(r0, r0);
1570     break;
1571   case Bytecodes::_i2f:
1572     __ scvtfws(v0, r0);
1573     break;
1574   case Bytecodes::_i2d:
1575     __ scvtfwd(v0, r0);
1576     break;
1577   case Bytecodes::_i2b:
1578     __ sxtbw(r0, r0);
1579     break;
1580   case Bytecodes::_i2c:
1581     __ uxthw(r0, r0);
1582     break;
1583   case Bytecodes::_i2s:
1584     __ sxthw(r0, r0);
1585     break;
1586   case Bytecodes::_l2i:
1587     __ uxtw(r0, r0);
1588     break;
1589   case Bytecodes::_l2f:
1590     __ scvtfs(v0, r0);
1591     break;
1592   case Bytecodes::_l2d:
1593     __ scvtfd(v0, r0);
1594     break;
1595   case Bytecodes::_f2i:
1596   {
1597     Label L_Okay;
1598     __ clear_fpsr();
1599     __ fcvtzsw(r0, v0);
1600     __ get_fpsr(r1);
1601     __ cbzw(r1, L_Okay);
1602     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2i));
1603     __ bind(L_Okay);
1604   }
1605     break;
1606   case Bytecodes::_f2l:
1607   {
1608     Label L_Okay;
1609     __ clear_fpsr();
1610     __ fcvtzs(r0, v0);
1611     __ get_fpsr(r1);
1612     __ cbzw(r1, L_Okay);
1613     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2l));
1614     __ bind(L_Okay);
1615   }
1616     break;
1617   case Bytecodes::_f2d:
1618     __ fcvts(v0, v0);
1619     break;
1620   case Bytecodes::_d2i:
1621   {
1622     Label L_Okay;
1623     __ clear_fpsr();
1624     __ fcvtzdw(r0, v0);
1625     __ get_fpsr(r1);
1626     __ cbzw(r1, L_Okay);
1627     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
1628     __ bind(L_Okay);
1629   }
1630     break;
1631   case Bytecodes::_d2l:
1632   {
1633     Label L_Okay;
1634     __ clear_fpsr();
1635     __ fcvtzd(r0, v0);
1636     __ get_fpsr(r1);
1637     __ cbzw(r1, L_Okay);
1638     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
1639     __ bind(L_Okay);
1640   }
1641     break;
1642   case Bytecodes::_d2f:
1643     __ fcvtd(v0, v0);
1644     break;
1645   default:
1646     ShouldNotReachHere();
1647   }
1648 }
1649 
1650 void TemplateTable::lcmp()
1651 {
1652   transition(ltos, itos);
1653   Label done;
1654   __ pop_l(r1);
1655   __ cmp(r1, r0);
1656   __ mov(r0, (u_int64_t)-1L);
1657   __ br(Assembler::LT, done);
1658   // __ mov(r0, 1UL);
1659   // __ csel(r0, r0, zr, Assembler::NE);
1660   // and here is a faster way
1661   __ csinc(r0, zr, zr, Assembler::EQ);
1662   __ bind(done);
1663 }
1664 
1665 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1666 {
1667   Label done;
1668   if (is_float) {
1669     // XXX get rid of pop here, use ... reg, mem32
1670     __ pop_f(v1);
1671     __ fcmps(v1, v0);
1672   } else {
1673     // XXX get rid of pop here, use ... reg, mem64
1674     __ pop_d(v1);
1675     __ fcmpd(v1, v0);
1676   }
1677   if (unordered_result < 0) {
1678     // we want -1 for unordered or less than, 0 for equal and 1 for
1679     // greater than.
1680     __ mov(r0, (u_int64_t)-1L);
1681     // for FP LT tests less than or unordered
1682     __ br(Assembler::LT, done);
1683     // install 0 for EQ otherwise 1
1684     __ csinc(r0, zr, zr, Assembler::EQ);
1685   } else {
1686     // we want -1 for less than, 0 for equal and 1 for unordered or
1687     // greater than.
1688     __ mov(r0, 1L);
1689     // for FP HI tests greater than or unordered
1690     __ br(Assembler::HI, done);
1691     // install 0 for EQ otherwise ~0
1692     __ csinv(r0, zr, zr, Assembler::EQ);
1693 
1694   }
1695   __ bind(done);
1696 }
1697 
1698 void TemplateTable::branch(bool is_jsr, bool is_wide)
1699 {
1700   // We might be moving to a safepoint.  The thread which calls
1701   // Interpreter::notice_safepoints() will effectively flush its cache
1702   // when it makes a system call, but we need to do something to
1703   // ensure that we see the changed dispatch table.
1704   __ membar(MacroAssembler::LoadLoad);
1705 
1706   __ profile_taken_branch(r0, r1);
1707   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1708                              InvocationCounter::counter_offset();
1709   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1710                               InvocationCounter::counter_offset();
1711 
1712   // load branch displacement
1713   if (!is_wide) {
1714     __ ldrh(r2, at_bcp(1));
1715     __ rev16(r2, r2);
1716     // sign extend the 16 bit value in r2
1717     __ sbfm(r2, r2, 0, 15);
1718   } else {
1719     __ ldrw(r2, at_bcp(1));
1720     __ revw(r2, r2);
1721     // sign extend the 32 bit value in r2
1722     __ sbfm(r2, r2, 0, 31);
1723   }
1724 
1725   // Handle all the JSR stuff here, then exit.
1726   // It's much shorter and cleaner than intermingling with the non-JSR
1727   // normal-branch stuff occurring below.
1728 
1729   if (is_jsr) {
1730     // Pre-load the next target bytecode into rscratch1
1731     __ load_unsigned_byte(rscratch1, Address(rbcp, r2));
1732     // compute return address as bci
1733     __ ldr(rscratch2, Address(rmethod, Method::const_offset()));
1734     __ add(rscratch2, rscratch2,
1735            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1736     __ sub(r1, rbcp, rscratch2);
1737     __ push_i(r1);
1738     // Adjust the bcp by the 16-bit displacement in r2
1739     __ add(rbcp, rbcp, r2);
1740     __ dispatch_only(vtos);
1741     return;
1742   }
1743 
1744   // Normal (non-jsr) branch handling
1745 
1746   // Adjust the bcp by the displacement in r2
1747   __ add(rbcp, rbcp, r2);
1748 
1749   assert(UseLoopCounter || !UseOnStackReplacement,
1750          "on-stack-replacement requires loop counters");
1751   Label backedge_counter_overflow;
1752   Label profile_method;
1753   Label dispatch;
1754   if (UseLoopCounter) {
1755     // increment backedge counter for backward branches
1756     // r0: MDO
1757     // w1: MDO bumped taken-count
1758     // r2: target offset
1759     __ cmp(r2, zr);
1760     __ br(Assembler::GT, dispatch); // count only if backward branch
1761 
1762     // ECN: FIXME: This code smells
1763     // check if MethodCounters exists
1764     Label has_counters;
1765     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1766     __ cbnz(rscratch1, has_counters);
1767     __ push(r0);
1768     __ push(r1);
1769     __ push(r2);
1770     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1771             InterpreterRuntime::build_method_counters), rmethod);
1772     __ pop(r2);
1773     __ pop(r1);
1774     __ pop(r0);
1775     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1776     __ cbz(rscratch1, dispatch); // No MethodCounters allocated, OutOfMemory
1777     __ bind(has_counters);
1778 
1779     if (TieredCompilation) {
1780       Label no_mdo;
1781       int increment = InvocationCounter::count_increment;
1782       int mask = ((1 << Tier0BackedgeNotifyFreqLog) - 1) << InvocationCounter::count_shift;
1783       if (ProfileInterpreter) {
1784         // Are we profiling?
1785         __ ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
1786         __ cbz(r1, no_mdo);
1787         // Increment the MDO backedge counter
1788         const Address mdo_backedge_counter(r1, in_bytes(MethodData::backedge_counter_offset()) +
1789                                            in_bytes(InvocationCounter::counter_offset()));
1790         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1791                                    r0, rscratch2, false, Assembler::EQ, &backedge_counter_overflow);
1792         __ b(dispatch);
1793       }
1794       __ bind(no_mdo);
1795       // Increment backedge counter in MethodCounters*
1796       __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1797       __ increment_mask_and_jump(Address(rscratch1, be_offset), increment, mask,
1798                                  r0, rscratch2, false, Assembler::EQ, &backedge_counter_overflow);
1799     } else {
1800       // increment counter
1801       __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
1802       __ ldrw(r0, Address(rscratch2, be_offset));        // load backedge counter
1803       __ addw(rscratch1, r0, InvocationCounter::count_increment); // increment counter
1804       __ strw(rscratch1, Address(rscratch2, be_offset));        // store counter
1805 
1806       __ ldrw(r0, Address(rscratch2, inv_offset));    // load invocation counter
1807       __ andw(r0, r0, (unsigned)InvocationCounter::count_mask_value); // and the status bits
1808       __ addw(r0, r0, rscratch1);        // add both counters
1809 
1810       if (ProfileInterpreter) {
1811         // Test to see if we should create a method data oop
1812         __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterProfileLimit));
1813         __ ldrw(rscratch1, rscratch1);
1814         __ cmpw(r0, rscratch1);
1815         __ br(Assembler::LT, dispatch);
1816 
1817         // if no method data exists, go to profile method
1818         __ test_method_data_pointer(r0, profile_method);
1819 
1820         if (UseOnStackReplacement) {
1821           // check for overflow against w1 which is the MDO taken count
1822           __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
1823           __ ldrw(rscratch1, rscratch1);
1824           __ cmpw(r1, rscratch1);
1825           __ br(Assembler::LO, dispatch); // Intel == Assembler::below
1826 
1827           // When ProfileInterpreter is on, the backedge_count comes
1828           // from the MethodData*, which value does not get reset on
1829           // the call to frequency_counter_overflow().  To avoid
1830           // excessive calls to the overflow routine while the method is
1831           // being compiled, add a second test to make sure the overflow
1832           // function is called only once every overflow_frequency.
1833           const int overflow_frequency = 1024;
1834           __ andsw(r1, r1, overflow_frequency - 1);
1835           __ br(Assembler::EQ, backedge_counter_overflow);
1836 
1837         }
1838       } else {
1839         if (UseOnStackReplacement) {
1840           // check for overflow against w0, which is the sum of the
1841           // counters
1842           __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
1843           __ ldrw(rscratch1, rscratch1);
1844           __ cmpw(r0, rscratch1);
1845           __ br(Assembler::HS, backedge_counter_overflow); // Intel == Assembler::aboveEqual
1846         }
1847       }
1848     }
1849   }
1850   __ bind(dispatch);
1851 
1852   // Pre-load the next target bytecode into rscratch1
1853   __ load_unsigned_byte(rscratch1, Address(rbcp, 0));
1854 
1855   // continue with the bytecode @ target
1856   // rscratch1: target bytecode
1857   // rbcp: target bcp
1858   __ dispatch_only(vtos);
1859 
1860   if (UseLoopCounter) {
1861     if (ProfileInterpreter) {
1862       // Out-of-line code to allocate method data oop.
1863       __ bind(profile_method);
1864       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1865       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1866       __ set_method_data_pointer_for_bcp();
1867       __ b(dispatch);
1868     }
1869 
1870     if (TieredCompilation || UseOnStackReplacement) {
1871       // invocation counter overflow
1872       __ bind(backedge_counter_overflow);
1873       __ neg(r2, r2);
1874       __ add(r2, r2, rbcp);     // branch bcp
1875       // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
1876       __ call_VM(noreg,
1877                  CAST_FROM_FN_PTR(address,
1878                                   InterpreterRuntime::frequency_counter_overflow),
1879                  r2);
1880       if (!UseOnStackReplacement)
1881         __ b(dispatch);
1882     }
1883 
1884     if (UseOnStackReplacement) {
1885       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1886 
1887       // r0: osr nmethod (osr ok) or NULL (osr not possible)
1888       // w1: target bytecode
1889       // r2: scratch
1890       __ cbz(r0, dispatch);     // test result -- no osr if null
1891       // nmethod may have been invalidated (VM may block upon call_VM return)
1892       __ ldrw(r2, Address(r0, nmethod::entry_bci_offset()));
1893       // InvalidOSREntryBci == -2 which overflows cmpw as unsigned
1894       // use cmnw against -InvalidOSREntryBci which does the same thing
1895       __ cmn(r2, -InvalidOSREntryBci);
1896       __ br(Assembler::EQ, dispatch);
1897 
1898       // We have the address of an on stack replacement routine in r0
1899       // We need to prepare to execute the OSR method. First we must
1900       // migrate the locals and monitors off of the stack.
1901 
1902       __ mov(r19, r0);                             // save the nmethod
1903 
1904       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
1905 
1906       // r0 is OSR buffer, move it to expected parameter location
1907       __ mov(j_rarg0, r0);
1908 
1909       // remove activation
1910       // get sender esp
1911       __ ldr(esp,
1912           Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
1913       // remove frame anchor
1914       __ leave();
1915       // Ensure compiled code always sees stack at proper alignment
1916       __ andr(sp, esp, -16);
1917 
1918       // and begin the OSR nmethod
1919       __ ldr(rscratch1, Address(r19, nmethod::osr_entry_point_offset()));
1920       __ br(rscratch1);
1921     }
1922   }
1923 }
1924 
1925 
1926 void TemplateTable::if_0cmp(Condition cc)
1927 {
1928   transition(itos, vtos);
1929   // assume branch is more often taken than not (loops use backward branches)
1930   Label not_taken;
1931   if (cc == equal)
1932     __ cbnzw(r0, not_taken);
1933   else if (cc == not_equal)
1934     __ cbzw(r0, not_taken);
1935   else {
1936     __ andsw(zr, r0, r0);
1937     __ br(j_not(cc), not_taken);
1938   }
1939 
1940   branch(false, false);
1941   __ bind(not_taken);
1942   __ profile_not_taken_branch(r0);
1943 }
1944 
1945 void TemplateTable::if_icmp(Condition cc)
1946 {
1947   transition(itos, vtos);
1948   // assume branch is more often taken than not (loops use backward branches)
1949   Label not_taken;
1950   __ pop_i(r1);
1951   __ cmpw(r1, r0, Assembler::LSL);
1952   __ br(j_not(cc), not_taken);
1953   branch(false, false);
1954   __ bind(not_taken);
1955   __ profile_not_taken_branch(r0);
1956 }
1957 
1958 void TemplateTable::if_nullcmp(Condition cc)
1959 {
1960   transition(atos, vtos);
1961   // assume branch is more often taken than not (loops use backward branches)
1962   Label not_taken;
1963   if (cc == equal)
1964     __ cbnz(r0, not_taken);
1965   else
1966     __ cbz(r0, not_taken);
1967   branch(false, false);
1968   __ bind(not_taken);
1969   __ profile_not_taken_branch(r0);
1970 }
1971 
1972 void TemplateTable::if_acmp(Condition cc)
1973 {
1974   transition(atos, vtos);
1975   // assume branch is more often taken than not (loops use backward branches)
1976   Label not_taken;
1977   __ pop_ptr(r1);
1978   __ cmp(r1, r0);
1979   __ br(j_not(cc), not_taken);
1980   branch(false, false);
1981   __ bind(not_taken);
1982   __ profile_not_taken_branch(r0);
1983 }
1984 
1985 void TemplateTable::ret() {
1986   transition(vtos, vtos);
1987   // We might be moving to a safepoint.  The thread which calls
1988   // Interpreter::notice_safepoints() will effectively flush its cache
1989   // when it makes a system call, but we need to do something to
1990   // ensure that we see the changed dispatch table.
1991   __ membar(MacroAssembler::LoadLoad);
1992 
1993   locals_index(r1);
1994   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
1995   __ profile_ret(r1, r2);
1996   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
1997   __ lea(rbcp, Address(rbcp, r1));
1998   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
1999   __ dispatch_next(vtos);
2000 }
2001 
2002 void TemplateTable::wide_ret() {
2003   transition(vtos, vtos);
2004   locals_index_wide(r1);
2005   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2006   __ profile_ret(r1, r2);
2007   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2008   __ lea(rbcp, Address(rbcp, r1));
2009   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2010   __ dispatch_next(vtos);
2011 }
2012 
2013 
2014 void TemplateTable::tableswitch() {
2015   Label default_case, continue_execution;
2016   transition(itos, vtos);
2017   // align rbcp
2018   __ lea(r1, at_bcp(BytesPerInt));
2019   __ andr(r1, r1, -BytesPerInt);
2020   // load lo & hi
2021   __ ldrw(r2, Address(r1, BytesPerInt));
2022   __ ldrw(r3, Address(r1, 2 * BytesPerInt));
2023   __ rev32(r2, r2);
2024   __ rev32(r3, r3);
2025   // check against lo & hi
2026   __ cmpw(r0, r2);
2027   __ br(Assembler::LT, default_case);
2028   __ cmpw(r0, r3);
2029   __ br(Assembler::GT, default_case);
2030   // lookup dispatch offset
2031   __ subw(r0, r0, r2);
2032   __ lea(r3, Address(r1, r0, Address::uxtw(2)));
2033   __ ldrw(r3, Address(r3, 3 * BytesPerInt));
2034   __ profile_switch_case(r0, r1, r2);
2035   // continue execution
2036   __ bind(continue_execution);
2037   __ rev32(r3, r3);
2038   __ load_unsigned_byte(rscratch1, Address(rbcp, r3, Address::sxtw(0)));
2039   __ add(rbcp, rbcp, r3, ext::sxtw);
2040   __ dispatch_only(vtos);
2041   // handle default
2042   __ bind(default_case);
2043   __ profile_switch_default(r0);
2044   __ ldrw(r3, Address(r1, 0));
2045   __ b(continue_execution);
2046 }
2047 
2048 void TemplateTable::lookupswitch() {
2049   transition(itos, itos);
2050   __ stop("lookupswitch bytecode should have been rewritten");
2051 }
2052 
2053 void TemplateTable::fast_linearswitch() {
2054   transition(itos, vtos);
2055   Label loop_entry, loop, found, continue_execution;
2056   // bswap r0 so we can avoid bswapping the table entries
2057   __ rev32(r0, r0);
2058   // align rbcp
2059   __ lea(r19, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2060                                     // this instruction (change offsets
2061                                     // below)
2062   __ andr(r19, r19, -BytesPerInt);
2063   // set counter
2064   __ ldrw(r1, Address(r19, BytesPerInt));
2065   __ rev32(r1, r1);
2066   __ b(loop_entry);
2067   // table search
2068   __ bind(loop);
2069   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2070   __ ldrw(rscratch1, Address(rscratch1, 2 * BytesPerInt));
2071   __ cmpw(r0, rscratch1);
2072   __ br(Assembler::EQ, found);
2073   __ bind(loop_entry);
2074   __ subs(r1, r1, 1);
2075   __ br(Assembler::PL, loop);
2076   // default case
2077   __ profile_switch_default(r0);
2078   __ ldrw(r3, Address(r19, 0));
2079   __ b(continue_execution);
2080   // entry found -> get offset
2081   __ bind(found);
2082   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2083   __ ldrw(r3, Address(rscratch1, 3 * BytesPerInt));
2084   __ profile_switch_case(r1, r0, r19);
2085   // continue execution
2086   __ bind(continue_execution);
2087   __ rev32(r3, r3);
2088   __ add(rbcp, rbcp, r3, ext::sxtw);
2089   __ ldrb(rscratch1, Address(rbcp, 0));
2090   __ dispatch_only(vtos);
2091 }
2092 
2093 void TemplateTable::fast_binaryswitch() {
2094   transition(itos, vtos);
2095   // Implementation using the following core algorithm:
2096   //
2097   // int binary_search(int key, LookupswitchPair* array, int n) {
2098   //   // Binary search according to "Methodik des Programmierens" by
2099   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2100   //   int i = 0;
2101   //   int j = n;
2102   //   while (i+1 < j) {
2103   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2104   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2105   //     // where a stands for the array and assuming that the (inexisting)
2106   //     // element a[n] is infinitely big.
2107   //     int h = (i + j) >> 1;
2108   //     // i < h < j
2109   //     if (key < array[h].fast_match()) {
2110   //       j = h;
2111   //     } else {
2112   //       i = h;
2113   //     }
2114   //   }
2115   //   // R: a[i] <= key < a[i+1] or Q
2116   //   // (i.e., if key is within array, i is the correct index)
2117   //   return i;
2118   // }
2119 
2120   // Register allocation
2121   const Register key   = r0; // already set (tosca)
2122   const Register array = r1;
2123   const Register i     = r2;
2124   const Register j     = r3;
2125   const Register h     = rscratch1;
2126   const Register temp  = rscratch2;
2127 
2128   // Find array start
2129   __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2130                                           // get rid of this
2131                                           // instruction (change
2132                                           // offsets below)
2133   __ andr(array, array, -BytesPerInt);
2134 
2135   // Initialize i & j
2136   __ mov(i, 0);                            // i = 0;
2137   __ ldrw(j, Address(array, -BytesPerInt)); // j = length(array);
2138 
2139   // Convert j into native byteordering
2140   __ rev32(j, j);
2141 
2142   // And start
2143   Label entry;
2144   __ b(entry);
2145 
2146   // binary search loop
2147   {
2148     Label loop;
2149     __ bind(loop);
2150     // int h = (i + j) >> 1;
2151     __ addw(h, i, j);                           // h = i + j;
2152     __ lsrw(h, h, 1);                                   // h = (i + j) >> 1;
2153     // if (key < array[h].fast_match()) {
2154     //   j = h;
2155     // } else {
2156     //   i = h;
2157     // }
2158     // Convert array[h].match to native byte-ordering before compare
2159     __ ldr(temp, Address(array, h, Address::lsl(3)));
2160     __ rev32(temp, temp);
2161     __ cmpw(key, temp);
2162     // j = h if (key <  array[h].fast_match())
2163     __ csel(j, h, j, Assembler::LT);
2164     // i = h if (key >= array[h].fast_match())
2165     __ csel(i, h, i, Assembler::GE);
2166     // while (i+1 < j)
2167     __ bind(entry);
2168     __ addw(h, i, 1);          // i+1
2169     __ cmpw(h, j);             // i+1 < j
2170     __ br(Assembler::LT, loop);
2171   }
2172 
2173   // end of binary search, result index is i (must check again!)
2174   Label default_case;
2175   // Convert array[i].match to native byte-ordering before compare
2176   __ ldr(temp, Address(array, i, Address::lsl(3)));
2177   __ rev32(temp, temp);
2178   __ cmpw(key, temp);
2179   __ br(Assembler::NE, default_case);
2180 
2181   // entry found -> j = offset
2182   __ add(j, array, i, ext::uxtx, 3);
2183   __ ldrw(j, Address(j, BytesPerInt));
2184   __ profile_switch_case(i, key, array);
2185   __ rev32(j, j);
2186   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2187   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2188   __ dispatch_only(vtos);
2189 
2190   // default case -> j = default offset
2191   __ bind(default_case);
2192   __ profile_switch_default(i);
2193   __ ldrw(j, Address(array, -2 * BytesPerInt));
2194   __ rev32(j, j);
2195   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2196   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2197   __ dispatch_only(vtos);
2198 }
2199 
2200 
2201 void TemplateTable::_return(TosState state)
2202 {
2203   transition(state, state);
2204   assert(_desc->calls_vm(),
2205          "inconsistent calls_vm information"); // call in remove_activation
2206 
2207   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2208     assert(state == vtos, "only valid state");
2209 
2210     __ ldr(c_rarg1, aaddress(0));
2211     __ load_klass(r3, c_rarg1);
2212     __ ldrw(r3, Address(r3, Klass::access_flags_offset()));
2213     __ tst(r3, JVM_ACC_HAS_FINALIZER);
2214     Label skip_register_finalizer;
2215     __ br(Assembler::EQ, skip_register_finalizer);
2216 
2217     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2218 
2219     __ bind(skip_register_finalizer);
2220   }
2221 
2222   // Issue a StoreStore barrier after all stores but before return
2223   // from any constructor for any class with a final field.  We don't
2224   // know if this is a finalizer, so we always do so.
2225   if (_desc->bytecode() == Bytecodes::_return)
2226     __ membar(MacroAssembler::StoreStore);
2227 
2228   // Narrow result if state is itos but result type is smaller.
2229   // Need to narrow in the return bytecode rather than in generate_return_entry
2230   // since compiled code callers expect the result to already be narrowed.
2231   if (state == itos) {
2232     __ narrow(r0);
2233   }
2234 
2235   __ remove_activation(state);
2236   __ ret(lr);
2237 }
2238 
2239 // ----------------------------------------------------------------------------
2240 // Volatile variables demand their effects be made known to all CPU's
2241 // in order.  Store buffers on most chips allow reads & writes to
2242 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2243 // without some kind of memory barrier (i.e., it's not sufficient that
2244 // the interpreter does not reorder volatile references, the hardware
2245 // also must not reorder them).
2246 //
2247 // According to the new Java Memory Model (JMM):
2248 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2249 //     writes act as aquire & release, so:
2250 // (2) A read cannot let unrelated NON-volatile memory refs that
2251 //     happen after the read float up to before the read.  It's OK for
2252 //     non-volatile memory refs that happen before the volatile read to
2253 //     float down below it.
2254 // (3) Similar a volatile write cannot let unrelated NON-volatile
2255 //     memory refs that happen BEFORE the write float down to after the
2256 //     write.  It's OK for non-volatile memory refs that happen after the
2257 //     volatile write to float up before it.
2258 //
2259 // We only put in barriers around volatile refs (they are expensive),
2260 // not _between_ memory refs (that would require us to track the
2261 // flavor of the previous memory refs).  Requirements (2) and (3)
2262 // require some barriers before volatile stores and after volatile
2263 // loads.  These nearly cover requirement (1) but miss the
2264 // volatile-store-volatile-load case.  This final case is placed after
2265 // volatile-stores although it could just as well go before
2266 // volatile-loads.
2267 
2268 void TemplateTable::resolve_cache_and_index(int byte_no,
2269                                             Register Rcache,
2270                                             Register index,
2271                                             size_t index_size) {
2272   const Register temp = r19;
2273   assert_different_registers(Rcache, index, temp);
2274 
2275   Label resolved;
2276   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2277   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2278   __ cmp(temp, (int) bytecode());  // have we resolved this bytecode?
2279   __ br(Assembler::EQ, resolved);
2280 
2281   // resolve first time through
2282   address entry;
2283   switch (bytecode()) {
2284   case Bytecodes::_getstatic:
2285   case Bytecodes::_putstatic:
2286   case Bytecodes::_getfield:
2287   case Bytecodes::_putfield:
2288     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_get_put);
2289     break;
2290   case Bytecodes::_invokevirtual:
2291   case Bytecodes::_invokespecial:
2292   case Bytecodes::_invokestatic:
2293   case Bytecodes::_invokeinterface:
2294     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invoke);
2295     break;
2296   case Bytecodes::_invokehandle:
2297     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invokehandle);
2298     break;
2299   case Bytecodes::_invokedynamic:
2300     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invokedynamic);
2301     break;
2302   default:
2303     fatal(err_msg("unexpected bytecode: %s", Bytecodes::name(bytecode())));
2304     break;
2305   }
2306   __ mov(temp, (int) bytecode());
2307   __ call_VM(noreg, entry, temp);
2308 
2309   // Update registers with resolved info
2310   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2311   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2312   // so all clients ofthis method must be modified accordingly
2313   __ bind(resolved);
2314 }
2315 
2316 // The Rcache and index registers must be set before call
2317 // n.b unlike x86 cache already includes the index offset
2318 void TemplateTable::load_field_cp_cache_entry(Register obj,
2319                                               Register cache,
2320                                               Register index,
2321                                               Register off,
2322                                               Register flags,
2323                                               bool is_static = false) {
2324   assert_different_registers(cache, index, flags, off);
2325 
2326   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2327   // Field offset
2328   __ ldr(off, Address(cache, in_bytes(cp_base_offset +
2329                                           ConstantPoolCacheEntry::f2_offset())));
2330   // Flags
2331   __ ldrw(flags, Address(cache, in_bytes(cp_base_offset +
2332                                            ConstantPoolCacheEntry::flags_offset())));
2333 
2334   // klass overwrite register
2335   if (is_static) {
2336     __ ldr(obj, Address(cache, in_bytes(cp_base_offset +
2337                                         ConstantPoolCacheEntry::f1_offset())));
2338     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2339     __ ldr(obj, Address(obj, mirror_offset));
2340   }
2341 }
2342 
2343 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2344                                                Register method,
2345                                                Register itable_index,
2346                                                Register flags,
2347                                                bool is_invokevirtual,
2348                                                bool is_invokevfinal, /*unused*/
2349                                                bool is_invokedynamic) {
2350   // setup registers
2351   const Register cache = rscratch2;
2352   const Register index = r4;
2353   assert_different_registers(method, flags);
2354   assert_different_registers(method, cache, index);
2355   assert_different_registers(itable_index, flags);
2356   assert_different_registers(itable_index, cache, index);
2357   // determine constant pool cache field offsets
2358   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2359   const int method_offset = in_bytes(
2360     ConstantPoolCache::base_offset() +
2361       (is_invokevirtual
2362        ? ConstantPoolCacheEntry::f2_offset()
2363        : ConstantPoolCacheEntry::f1_offset()));
2364   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2365                                     ConstantPoolCacheEntry::flags_offset());
2366   // access constant pool cache fields
2367   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2368                                     ConstantPoolCacheEntry::f2_offset());
2369 
2370   size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2371   resolve_cache_and_index(byte_no, cache, index, index_size);
2372   __ ldr(method, Address(cache, method_offset));
2373 
2374   if (itable_index != noreg) {
2375     __ ldr(itable_index, Address(cache, index_offset));
2376   }
2377   __ ldrw(flags, Address(cache, flags_offset));
2378 }
2379 
2380 
2381 // The registers cache and index expected to be set before call.
2382 // Correct values of the cache and index registers are preserved.
2383 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2384                                             bool is_static, bool has_tos) {
2385   // do the JVMTI work here to avoid disturbing the register state below
2386   // We use c_rarg registers here because we want to use the register used in
2387   // the call to the VM
2388   if (JvmtiExport::can_post_field_access()) {
2389     // Check to see if a field access watch has been set before we
2390     // take the time to call into the VM.
2391     Label L1;
2392     assert_different_registers(cache, index, r0);
2393     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2394     __ ldrw(r0, Address(rscratch1));
2395     __ cbzw(r0, L1);
2396 
2397     __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2398     __ lea(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2399 
2400     if (is_static) {
2401       __ mov(c_rarg1, zr); // NULL object reference
2402     } else {
2403       __ ldr(c_rarg1, at_tos()); // get object pointer without popping it
2404       __ verify_oop(c_rarg1);
2405     }
2406     // c_rarg1: object pointer or NULL
2407     // c_rarg2: cache entry pointer
2408     // c_rarg3: jvalue object on the stack
2409     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2410                                        InterpreterRuntime::post_field_access),
2411                c_rarg1, c_rarg2, c_rarg3);
2412     __ get_cache_and_index_at_bcp(cache, index, 1);
2413     __ bind(L1);
2414   }
2415 }
2416 
2417 void TemplateTable::pop_and_check_object(Register r)
2418 {
2419   __ pop_ptr(r);
2420   __ null_check(r);  // for field access must check obj.
2421   __ verify_oop(r);
2422 }
2423 
2424 void TemplateTable::getfield_or_static(int byte_no, bool is_static)
2425 {
2426   const Register cache = r2;
2427   const Register index = r3;
2428   const Register obj   = r4;
2429   const Register off   = r19;
2430   const Register flags = r0;
2431   const Register raw_flags = r6;
2432   const Register bc    = r4; // uses same reg as obj, so don't mix them
2433 
2434   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2435   jvmti_post_field_access(cache, index, is_static, false);
2436   load_field_cp_cache_entry(obj, cache, index, off, raw_flags, is_static);
2437 
2438   if (!is_static) {
2439     // obj is on the stack
2440     pop_and_check_object(obj);
2441   }
2442 
2443   // 8179954: We need to make sure that the code generated for
2444   // volatile accesses forms a sequentially-consistent set of
2445   // operations when combined with STLR and LDAR.  Without a leading
2446   // membar it's possible for a simple Dekker test to fail if loads
2447   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
2448   // the stores in one method and we interpret the loads in another.
2449   if (! UseBarriersForVolatile) {
2450     Label notVolatile;
2451     __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2452     __ membar(MacroAssembler::AnyAny);
2453     __ bind(notVolatile);
2454   }
2455 
2456   const Address field(obj, off);
2457 
2458   Label Done, notByte, notBool, notInt, notShort, notChar,
2459               notLong, notFloat, notObj, notDouble;
2460 
2461   // x86 uses a shift and mask or wings it with a shift plus assert
2462   // the mask is not needed. aarch64 just uses bitfield extract
2463   __ ubfxw(flags, raw_flags, ConstantPoolCacheEntry::tos_state_shift,
2464            ConstantPoolCacheEntry::tos_state_bits);
2465 
2466   assert(btos == 0, "change code, btos != 0");
2467   __ cbnz(flags, notByte);
2468 
2469   // btos
2470   __ load_signed_byte(r0, field);
2471   __ push(btos);
2472   // Rewrite bytecode to be faster
2473   if (!is_static) {
2474     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2475   }
2476   __ b(Done);
2477 
2478   __ bind(notByte);
2479   __ cmp(flags, ztos);
2480   __ br(Assembler::NE, notBool);
2481 
2482   // ztos (same code as btos)
2483   __ ldrsb(r0, field);
2484   __ push(ztos);
2485   // Rewrite bytecode to be faster
2486   if (!is_static) {
2487     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2488     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2489   }
2490   __ b(Done);
2491 
2492   __ bind(notBool);
2493   __ cmp(flags, atos);
2494   __ br(Assembler::NE, notObj);
2495   // atos
2496   __ load_heap_oop(r0, field);
2497   __ push(atos);
2498   if (!is_static) {
2499     patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2500   }
2501   __ b(Done);
2502 
2503   __ bind(notObj);
2504   __ cmp(flags, itos);
2505   __ br(Assembler::NE, notInt);
2506   // itos
2507   __ ldrw(r0, field);
2508   __ push(itos);
2509   // Rewrite bytecode to be faster
2510   if (!is_static) {
2511     patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2512   }
2513   __ b(Done);
2514 
2515   __ bind(notInt);
2516   __ cmp(flags, ctos);
2517   __ br(Assembler::NE, notChar);
2518   // ctos
2519   __ load_unsigned_short(r0, field);
2520   __ push(ctos);
2521   // Rewrite bytecode to be faster
2522   if (!is_static) {
2523     patch_bytecode(Bytecodes::_fast_cgetfield, bc, r1);
2524   }
2525   __ b(Done);
2526 
2527   __ bind(notChar);
2528   __ cmp(flags, stos);
2529   __ br(Assembler::NE, notShort);
2530   // stos
2531   __ load_signed_short(r0, field);
2532   __ push(stos);
2533   // Rewrite bytecode to be faster
2534   if (!is_static) {
2535     patch_bytecode(Bytecodes::_fast_sgetfield, bc, r1);
2536   }
2537   __ b(Done);
2538 
2539   __ bind(notShort);
2540   __ cmp(flags, ltos);
2541   __ br(Assembler::NE, notLong);
2542   // ltos
2543   __ ldr(r0, field);
2544   __ push(ltos);
2545   // Rewrite bytecode to be faster
2546   if (!is_static) {
2547     patch_bytecode(Bytecodes::_fast_lgetfield, bc, r1);
2548   }
2549   __ b(Done);
2550 
2551   __ bind(notLong);
2552   __ cmp(flags, ftos);
2553   __ br(Assembler::NE, notFloat);
2554   // ftos
2555   __ ldrs(v0, field);
2556   __ push(ftos);
2557   // Rewrite bytecode to be faster
2558   if (!is_static) {
2559     patch_bytecode(Bytecodes::_fast_fgetfield, bc, r1);
2560   }
2561   __ b(Done);
2562 
2563   __ bind(notFloat);
2564 #ifdef ASSERT
2565   __ cmp(flags, dtos);
2566   __ br(Assembler::NE, notDouble);
2567 #endif
2568   // dtos
2569   __ ldrd(v0, field);
2570   __ push(dtos);
2571   // Rewrite bytecode to be faster
2572   if (!is_static) {
2573     patch_bytecode(Bytecodes::_fast_dgetfield, bc, r1);
2574   }
2575 #ifdef ASSERT
2576   __ b(Done);
2577 
2578   __ bind(notDouble);
2579   __ stop("Bad state");
2580 #endif
2581 
2582   __ bind(Done);
2583 
2584   Label notVolatile;
2585   __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2586   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2587   __ bind(notVolatile);
2588 }
2589 
2590 
2591 void TemplateTable::getfield(int byte_no)
2592 {
2593   getfield_or_static(byte_no, false);
2594 }
2595 
2596 void TemplateTable::getstatic(int byte_no)
2597 {
2598   getfield_or_static(byte_no, true);
2599 }
2600 
2601 // The registers cache and index expected to be set before call.
2602 // The function may destroy various registers, just not the cache and index registers.
2603 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2604   transition(vtos, vtos);
2605 
2606   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2607 
2608   if (JvmtiExport::can_post_field_modification()) {
2609     // Check to see if a field modification watch has been set before
2610     // we take the time to call into the VM.
2611     Label L1;
2612     assert_different_registers(cache, index, r0);
2613     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2614     __ ldrw(r0, Address(rscratch1));
2615     __ cbz(r0, L1);
2616 
2617     __ get_cache_and_index_at_bcp(c_rarg2, rscratch1, 1);
2618 
2619     if (is_static) {
2620       // Life is simple.  Null out the object pointer.
2621       __ mov(c_rarg1, zr);
2622     } else {
2623       // Life is harder. The stack holds the value on top, followed by
2624       // the object.  We don't know the size of the value, though; it
2625       // could be one or two words depending on its type. As a result,
2626       // we must find the type to determine where the object is.
2627       __ ldrw(c_rarg3, Address(c_rarg2,
2628                                in_bytes(cp_base_offset +
2629                                         ConstantPoolCacheEntry::flags_offset())));
2630       __ lsr(c_rarg3, c_rarg3,
2631              ConstantPoolCacheEntry::tos_state_shift);
2632       ConstantPoolCacheEntry::verify_tos_state_shift();
2633       Label nope2, done, ok;
2634       __ ldr(c_rarg1, at_tos_p1());  // initially assume a one word jvalue
2635       __ cmpw(c_rarg3, ltos);
2636       __ br(Assembler::EQ, ok);
2637       __ cmpw(c_rarg3, dtos);
2638       __ br(Assembler::NE, nope2);
2639       __ bind(ok);
2640       __ ldr(c_rarg1, at_tos_p2()); // ltos (two word jvalue)
2641       __ bind(nope2);
2642     }
2643     // cache entry pointer
2644     __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2645     // object (tos)
2646     __ mov(c_rarg3, esp);
2647     // c_rarg1: object pointer set up above (NULL if static)
2648     // c_rarg2: cache entry pointer
2649     // c_rarg3: jvalue object on the stack
2650     __ call_VM(noreg,
2651                CAST_FROM_FN_PTR(address,
2652                                 InterpreterRuntime::post_field_modification),
2653                c_rarg1, c_rarg2, c_rarg3);
2654     __ get_cache_and_index_at_bcp(cache, index, 1);
2655     __ bind(L1);
2656   }
2657 }
2658 
2659 void TemplateTable::putfield_or_static(int byte_no, bool is_static) {
2660   transition(vtos, vtos);
2661 
2662   const Register cache = r2;
2663   const Register index = r3;
2664   const Register obj   = r2;
2665   const Register off   = r19;
2666   const Register flags = r0;
2667   const Register bc    = r4;
2668 
2669   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2670   jvmti_post_field_mod(cache, index, is_static);
2671   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2672 
2673   Label Done;
2674   __ mov(r5, flags);
2675 
2676   {
2677     Label notVolatile;
2678     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2679     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2680     __ bind(notVolatile);
2681   }
2682 
2683   // field address
2684   const Address field(obj, off);
2685 
2686   Label notByte, notBool, notInt, notShort, notChar,
2687         notLong, notFloat, notObj, notDouble;
2688 
2689   // x86 uses a shift and mask or wings it with a shift plus assert
2690   // the mask is not needed. aarch64 just uses bitfield extract
2691   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
2692 
2693   assert(btos == 0, "change code, btos != 0");
2694   __ cbnz(flags, notByte);
2695 
2696   // btos
2697   {
2698     __ pop(btos);
2699     if (!is_static) pop_and_check_object(obj);
2700     __ strb(r0, field);
2701     if (!is_static) {
2702       patch_bytecode(Bytecodes::_fast_bputfield, bc, r1, true, byte_no);
2703     }
2704     __ b(Done);
2705   }
2706 
2707   __ bind(notByte);
2708   __ cmp(flags, ztos);
2709   __ br(Assembler::NE, notBool);
2710 
2711   // ztos
2712   {
2713     __ pop(ztos);
2714     if (!is_static) pop_and_check_object(obj);
2715     __ andw(r0, r0, 0x1);
2716     __ strb(r0, field);
2717     if (!is_static) {
2718       patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
2719     }
2720     __ b(Done);
2721   }
2722 
2723   __ bind(notBool);
2724   __ cmp(flags, atos);
2725   __ br(Assembler::NE, notObj);
2726 
2727   // atos
2728   {
2729     __ pop(atos);
2730     if (!is_static) pop_and_check_object(obj);
2731     // Store into the field
2732     do_oop_store(_masm, field, r0, _bs->kind(), false);
2733     if (!is_static) {
2734       patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
2735     }
2736     __ b(Done);
2737   }
2738 
2739   __ bind(notObj);
2740   __ cmp(flags, itos);
2741   __ br(Assembler::NE, notInt);
2742 
2743   // itos
2744   {
2745     __ pop(itos);
2746     if (!is_static) pop_and_check_object(obj);
2747     __ strw(r0, field);
2748     if (!is_static) {
2749       patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
2750     }
2751     __ b(Done);
2752   }
2753 
2754   __ bind(notInt);
2755   __ cmp(flags, ctos);
2756   __ br(Assembler::NE, notChar);
2757 
2758   // ctos
2759   {
2760     __ pop(ctos);
2761     if (!is_static) pop_and_check_object(obj);
2762     __ strh(r0, field);
2763     if (!is_static) {
2764       patch_bytecode(Bytecodes::_fast_cputfield, bc, r1, true, byte_no);
2765     }
2766     __ b(Done);
2767   }
2768 
2769   __ bind(notChar);
2770   __ cmp(flags, stos);
2771   __ br(Assembler::NE, notShort);
2772 
2773   // stos
2774   {
2775     __ pop(stos);
2776     if (!is_static) pop_and_check_object(obj);
2777     __ strh(r0, field);
2778     if (!is_static) {
2779       patch_bytecode(Bytecodes::_fast_sputfield, bc, r1, true, byte_no);
2780     }
2781     __ b(Done);
2782   }
2783 
2784   __ bind(notShort);
2785   __ cmp(flags, ltos);
2786   __ br(Assembler::NE, notLong);
2787 
2788   // ltos
2789   {
2790     __ pop(ltos);
2791     if (!is_static) pop_and_check_object(obj);
2792     __ str(r0, field);
2793     if (!is_static) {
2794       patch_bytecode(Bytecodes::_fast_lputfield, bc, r1, true, byte_no);
2795     }
2796     __ b(Done);
2797   }
2798 
2799   __ bind(notLong);
2800   __ cmp(flags, ftos);
2801   __ br(Assembler::NE, notFloat);
2802 
2803   // ftos
2804   {
2805     __ pop(ftos);
2806     if (!is_static) pop_and_check_object(obj);
2807     __ strs(v0, field);
2808     if (!is_static) {
2809       patch_bytecode(Bytecodes::_fast_fputfield, bc, r1, true, byte_no);
2810     }
2811     __ b(Done);
2812   }
2813 
2814   __ bind(notFloat);
2815 #ifdef ASSERT
2816   __ cmp(flags, dtos);
2817   __ br(Assembler::NE, notDouble);
2818 #endif
2819 
2820   // dtos
2821   {
2822     __ pop(dtos);
2823     if (!is_static) pop_and_check_object(obj);
2824     __ strd(v0, field);
2825     if (!is_static) {
2826       patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
2827     }
2828   }
2829 
2830 #ifdef ASSERT
2831   __ b(Done);
2832 
2833   __ bind(notDouble);
2834   __ stop("Bad state");
2835 #endif
2836 
2837   __ bind(Done);
2838 
2839   {
2840     Label notVolatile;
2841     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2842     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
2843     __ bind(notVolatile);
2844   }
2845 }
2846 
2847 void TemplateTable::putfield(int byte_no)
2848 {
2849   putfield_or_static(byte_no, false);
2850 }
2851 
2852 void TemplateTable::putstatic(int byte_no) {
2853   putfield_or_static(byte_no, true);
2854 }
2855 
2856 void TemplateTable::jvmti_post_fast_field_mod()
2857 {
2858   if (JvmtiExport::can_post_field_modification()) {
2859     // Check to see if a field modification watch has been set before
2860     // we take the time to call into the VM.
2861     Label L2;
2862     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2863     __ ldrw(c_rarg3, Address(rscratch1));
2864     __ cbzw(c_rarg3, L2);
2865     __ pop_ptr(r19);                  // copy the object pointer from tos
2866     __ verify_oop(r19);
2867     __ push_ptr(r19);                 // put the object pointer back on tos
2868     // Save tos values before call_VM() clobbers them. Since we have
2869     // to do it for every data type, we use the saved values as the
2870     // jvalue object.
2871     switch (bytecode()) {          // load values into the jvalue object
2872     case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
2873     case Bytecodes::_fast_bputfield: // fall through
2874     case Bytecodes::_fast_zputfield: // fall through
2875     case Bytecodes::_fast_sputfield: // fall through
2876     case Bytecodes::_fast_cputfield: // fall through
2877     case Bytecodes::_fast_iputfield: __ push_i(r0); break;
2878     case Bytecodes::_fast_dputfield: __ push_d(); break;
2879     case Bytecodes::_fast_fputfield: __ push_f(); break;
2880     case Bytecodes::_fast_lputfield: __ push_l(r0); break;
2881 
2882     default:
2883       ShouldNotReachHere();
2884     }
2885     __ mov(c_rarg3, esp);             // points to jvalue on the stack
2886     // access constant pool cache entry
2887     __ get_cache_entry_pointer_at_bcp(c_rarg2, r0, 1);
2888     __ verify_oop(r19);
2889     // r19: object pointer copied above
2890     // c_rarg2: cache entry pointer
2891     // c_rarg3: jvalue object on the stack
2892     __ call_VM(noreg,
2893                CAST_FROM_FN_PTR(address,
2894                                 InterpreterRuntime::post_field_modification),
2895                r19, c_rarg2, c_rarg3);
2896 
2897     switch (bytecode()) {             // restore tos values
2898     case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
2899     case Bytecodes::_fast_bputfield: // fall through
2900     case Bytecodes::_fast_zputfield: // fall through
2901     case Bytecodes::_fast_sputfield: // fall through
2902     case Bytecodes::_fast_cputfield: // fall through
2903     case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
2904     case Bytecodes::_fast_dputfield: __ pop_d(); break;
2905     case Bytecodes::_fast_fputfield: __ pop_f(); break;
2906     case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
2907     }
2908     __ bind(L2);
2909   }
2910 }
2911 
2912 void TemplateTable::fast_storefield(TosState state)
2913 {
2914   transition(state, vtos);
2915 
2916   ByteSize base = ConstantPoolCache::base_offset();
2917 
2918   jvmti_post_fast_field_mod();
2919 
2920   // access constant pool cache
2921   __ get_cache_and_index_at_bcp(r2, r1, 1);
2922 
2923   // Must prevent reordering of the following cp cache loads with bytecode load
2924   __ membar(MacroAssembler::LoadLoad);
2925 
2926   // test for volatile with r3
2927   __ ldrw(r3, Address(r2, in_bytes(base +
2928                                    ConstantPoolCacheEntry::flags_offset())));
2929 
2930   // replace index with field offset from cache entry
2931   __ ldr(r1, Address(r2, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
2932 
2933   {
2934     Label notVolatile;
2935     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2936     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2937     __ bind(notVolatile);
2938   }
2939 
2940   Label notVolatile;
2941 
2942   // Get object from stack
2943   pop_and_check_object(r2);
2944 
2945   // field address
2946   const Address field(r2, r1);
2947 
2948   // access field
2949   switch (bytecode()) {
2950   case Bytecodes::_fast_aputfield:
2951     do_oop_store(_masm, field, r0, _bs->kind(), false);
2952     break;
2953   case Bytecodes::_fast_lputfield:
2954     __ str(r0, field);
2955     break;
2956   case Bytecodes::_fast_iputfield:
2957     __ strw(r0, field);
2958     break;
2959   case Bytecodes::_fast_zputfield:
2960     __ andw(r0, r0, 0x1);  // boolean is true if LSB is 1
2961     // fall through to bputfield
2962   case Bytecodes::_fast_bputfield:
2963     __ strb(r0, field);
2964     break;
2965   case Bytecodes::_fast_sputfield:
2966     // fall through
2967   case Bytecodes::_fast_cputfield:
2968     __ strh(r0, field);
2969     break;
2970   case Bytecodes::_fast_fputfield:
2971     __ strs(v0, field);
2972     break;
2973   case Bytecodes::_fast_dputfield:
2974     __ strd(v0, field);
2975     break;
2976   default:
2977     ShouldNotReachHere();
2978   }
2979 
2980   {
2981     Label notVolatile;
2982     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2983     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
2984     __ bind(notVolatile);
2985   }
2986 }
2987 
2988 
2989 void TemplateTable::fast_accessfield(TosState state)
2990 {
2991   transition(atos, state);
2992   // Do the JVMTI work here to avoid disturbing the register state below
2993   if (JvmtiExport::can_post_field_access()) {
2994     // Check to see if a field access watch has been set before we
2995     // take the time to call into the VM.
2996     Label L1;
2997     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2998     __ ldrw(r2, Address(rscratch1));
2999     __ cbzw(r2, L1);
3000     // access constant pool cache entry
3001     __ get_cache_entry_pointer_at_bcp(c_rarg2, rscratch2, 1);
3002     __ verify_oop(r0);
3003     __ push_ptr(r0);  // save object pointer before call_VM() clobbers it
3004     __ mov(c_rarg1, r0);
3005     // c_rarg1: object pointer copied above
3006     // c_rarg2: cache entry pointer
3007     __ call_VM(noreg,
3008                CAST_FROM_FN_PTR(address,
3009                                 InterpreterRuntime::post_field_access),
3010                c_rarg1, c_rarg2);
3011     __ pop_ptr(r0); // restore object pointer
3012     __ bind(L1);
3013   }
3014 
3015   // access constant pool cache
3016   __ get_cache_and_index_at_bcp(r2, r1, 1);
3017 
3018   // Must prevent reordering of the following cp cache loads with bytecode load
3019   __ membar(MacroAssembler::LoadLoad);
3020 
3021   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3022                                   ConstantPoolCacheEntry::f2_offset())));
3023   __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3024                                    ConstantPoolCacheEntry::flags_offset())));
3025 
3026   // r0: object
3027   __ verify_oop(r0);
3028   __ null_check(r0);
3029   const Address field(r0, r1);
3030 
3031   // 8179954: We need to make sure that the code generated for
3032   // volatile accesses forms a sequentially-consistent set of
3033   // operations when combined with STLR and LDAR.  Without a leading
3034   // membar it's possible for a simple Dekker test to fail if loads
3035   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3036   // the stores in one method and we interpret the loads in another.
3037   if (! UseBarriersForVolatile) {
3038     Label notVolatile;
3039     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3040     __ membar(MacroAssembler::AnyAny);
3041     __ bind(notVolatile);
3042   }
3043 
3044   // access field
3045   switch (bytecode()) {
3046   case Bytecodes::_fast_agetfield:
3047     __ load_heap_oop(r0, field);
3048     __ verify_oop(r0);
3049     break;
3050   case Bytecodes::_fast_lgetfield:
3051     __ ldr(r0, field);
3052     break;
3053   case Bytecodes::_fast_igetfield:
3054     __ ldrw(r0, field);
3055     break;
3056   case Bytecodes::_fast_bgetfield:
3057     __ load_signed_byte(r0, field);
3058     break;
3059   case Bytecodes::_fast_sgetfield:
3060     __ load_signed_short(r0, field);
3061     break;
3062   case Bytecodes::_fast_cgetfield:
3063     __ load_unsigned_short(r0, field);
3064     break;
3065   case Bytecodes::_fast_fgetfield:
3066     __ ldrs(v0, field);
3067     break;
3068   case Bytecodes::_fast_dgetfield:
3069     __ ldrd(v0, field);
3070     break;
3071   default:
3072     ShouldNotReachHere();
3073   }
3074   {
3075     Label notVolatile;
3076     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3077     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3078     __ bind(notVolatile);
3079   }
3080 }
3081 
3082 void TemplateTable::fast_xaccess(TosState state)
3083 {
3084   transition(vtos, state);
3085 
3086   // get receiver
3087   __ ldr(r0, aaddress(0));
3088   // access constant pool cache
3089   __ get_cache_and_index_at_bcp(r2, r3, 2);
3090   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3091                                   ConstantPoolCacheEntry::f2_offset())));
3092 
3093   // 8179954: We need to make sure that the code generated for
3094   // volatile accesses forms a sequentially-consistent set of
3095   // operations when combined with STLR and LDAR.  Without a leading
3096   // membar it's possible for a simple Dekker test to fail if loads
3097   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3098   // the stores in one method and we interpret the loads in another.
3099   if (! UseBarriersForVolatile) {
3100     Label notVolatile;
3101     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3102                                      ConstantPoolCacheEntry::flags_offset())));
3103     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3104     __ membar(MacroAssembler::AnyAny);
3105     __ bind(notVolatile);
3106   }
3107 
3108   // make sure exception is reported in correct bcp range (getfield is
3109   // next instruction)
3110   __ increment(rbcp);
3111   __ null_check(r0);
3112   switch (state) {
3113   case itos:
3114     __ ldrw(r0, Address(r0, r1, Address::lsl(0)));
3115     break;
3116   case atos:
3117     __ load_heap_oop(r0, Address(r0, r1, Address::lsl(0)));
3118     __ verify_oop(r0);
3119     break;
3120   case ftos:
3121     __ ldrs(v0, Address(r0, r1, Address::lsl(0)));
3122     break;
3123   default:
3124     ShouldNotReachHere();
3125   }
3126 
3127   {
3128     Label notVolatile;
3129     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3130                                      ConstantPoolCacheEntry::flags_offset())));
3131     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3132     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3133     __ bind(notVolatile);
3134   }
3135 
3136   __ decrement(rbcp);
3137 }
3138 
3139 
3140 
3141 //-----------------------------------------------------------------------------
3142 // Calls
3143 
3144 void TemplateTable::count_calls(Register method, Register temp)
3145 {
3146   __ call_Unimplemented();
3147 }
3148 
3149 void TemplateTable::prepare_invoke(int byte_no,
3150                                    Register method, // linked method (or i-klass)
3151                                    Register index,  // itable index, MethodType, etc.
3152                                    Register recv,   // if caller wants to see it
3153                                    Register flags   // if caller wants to test it
3154                                    ) {
3155   // determine flags
3156   Bytecodes::Code code = bytecode();
3157   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3158   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3159   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3160   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3161   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3162   const bool load_receiver       = (recv  != noreg);
3163   const bool save_flags          = (flags != noreg);
3164   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3165   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3166   assert(flags == noreg || flags == r3, "");
3167   assert(recv  == noreg || recv  == r2, "");
3168 
3169   // setup registers & access constant pool cache
3170   if (recv  == noreg)  recv  = r2;
3171   if (flags == noreg)  flags = r3;
3172   assert_different_registers(method, index, recv, flags);
3173 
3174   // save 'interpreter return address'
3175   __ save_bcp();
3176 
3177   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3178 
3179   // maybe push appendix to arguments (just before return address)
3180   if (is_invokedynamic || is_invokehandle) {
3181     Label L_no_push;
3182     __ tbz(flags, ConstantPoolCacheEntry::has_appendix_shift, L_no_push);
3183     // Push the appendix as a trailing parameter.
3184     // This must be done before we get the receiver,
3185     // since the parameter_size includes it.
3186     __ push(r19);
3187     __ mov(r19, index);
3188     assert(ConstantPoolCacheEntry::_indy_resolved_references_appendix_offset == 0, "appendix expected at index+0");
3189     __ load_resolved_reference_at_index(index, r19);
3190     __ pop(r19);
3191     __ push(index);  // push appendix (MethodType, CallSite, etc.)
3192     __ bind(L_no_push);
3193   }
3194 
3195   // load receiver if needed (note: no return address pushed yet)
3196   if (load_receiver) {
3197     __ andw(recv, flags, ConstantPoolCacheEntry::parameter_size_mask);
3198     // FIXME -- is this actually correct? looks like it should be 2
3199     // const int no_return_pc_pushed_yet = -1;  // argument slot correction before we push return address
3200     // const int receiver_is_at_end      = -1;  // back off one slot to get receiver
3201     // Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
3202     // __ movptr(recv, recv_addr);
3203     __ add(rscratch1, esp, recv, ext::uxtx, 3); // FIXME: uxtb here?
3204     __ ldr(recv, Address(rscratch1, -Interpreter::expr_offset_in_bytes(1)));
3205     __ verify_oop(recv);
3206   }
3207 
3208   // compute return type
3209   // x86 uses a shift and mask or wings it with a shift plus assert
3210   // the mask is not needed. aarch64 just uses bitfield extract
3211   __ ubfxw(rscratch2, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
3212   // load return address
3213   {
3214     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3215     __ mov(rscratch1, table_addr);
3216     __ ldr(lr, Address(rscratch1, rscratch2, Address::lsl(3)));
3217   }
3218 }
3219 
3220 
3221 void TemplateTable::invokevirtual_helper(Register index,
3222                                          Register recv,
3223                                          Register flags)
3224 {
3225   // Uses temporary registers r0, r3
3226   assert_different_registers(index, recv, r0, r3);
3227   // Test for an invoke of a final method
3228   Label notFinal;
3229   __ tbz(flags, ConstantPoolCacheEntry::is_vfinal_shift, notFinal);
3230 
3231   const Register method = index;  // method must be rmethod
3232   assert(method == rmethod,
3233          "methodOop must be rmethod for interpreter calling convention");
3234 
3235   // do the call - the index is actually the method to call
3236   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3237 
3238   // It's final, need a null check here!
3239   __ null_check(recv);
3240 
3241   // profile this call
3242   __ profile_final_call(r0);
3243   __ profile_arguments_type(r0, method, r4, true);
3244 
3245   __ jump_from_interpreted(method, r0);
3246 
3247   __ bind(notFinal);
3248 
3249   // get receiver klass
3250   __ null_check(recv, oopDesc::klass_offset_in_bytes());
3251   __ load_klass(r0, recv);
3252 
3253   // profile this call
3254   __ profile_virtual_call(r0, rlocals, r3);
3255 
3256   // get target methodOop & entry point
3257   __ lookup_virtual_method(r0, index, method);
3258   __ profile_arguments_type(r3, method, r4, true);
3259   // FIXME -- this looks completely redundant. is it?
3260   // __ ldr(r3, Address(method, Method::interpreter_entry_offset()));
3261   __ jump_from_interpreted(method, r3);
3262 }
3263 
3264 void TemplateTable::invokevirtual(int byte_no)
3265 {
3266   transition(vtos, vtos);
3267   assert(byte_no == f2_byte, "use this argument");
3268 
3269   prepare_invoke(byte_no, rmethod, noreg, r2, r3);
3270 
3271   // rmethod: index (actually a Method*)
3272   // r2: receiver
3273   // r3: flags
3274 
3275   invokevirtual_helper(rmethod, r2, r3);
3276 }
3277 
3278 void TemplateTable::invokespecial(int byte_no)
3279 {
3280   transition(vtos, vtos);
3281   assert(byte_no == f1_byte, "use this argument");
3282 
3283   prepare_invoke(byte_no, rmethod, noreg,  // get f1 Method*
3284                  r2);  // get receiver also for null check
3285   __ verify_oop(r2);
3286   __ null_check(r2);
3287   // do the call
3288   __ profile_call(r0);
3289   __ profile_arguments_type(r0, rmethod, rbcp, false);
3290   __ jump_from_interpreted(rmethod, r0);
3291 }
3292 
3293 void TemplateTable::invokestatic(int byte_no)
3294 {
3295   transition(vtos, vtos);
3296   assert(byte_no == f1_byte, "use this argument");
3297 
3298   prepare_invoke(byte_no, rmethod);  // get f1 Method*
3299   // do the call
3300   __ profile_call(r0);
3301   __ profile_arguments_type(r0, rmethod, r4, false);
3302   __ jump_from_interpreted(rmethod, r0);
3303 }
3304 
3305 void TemplateTable::fast_invokevfinal(int byte_no)
3306 {
3307   __ call_Unimplemented();
3308 }
3309 
3310 void TemplateTable::invokeinterface(int byte_no) {
3311   transition(vtos, vtos);
3312   assert(byte_no == f1_byte, "use this argument");
3313 
3314   prepare_invoke(byte_no, r0, rmethod,  // get f1 Klass*, f2 Method*
3315                  r2, r3); // recv, flags
3316 
3317   // r0: interface klass (from f1)
3318   // rmethod: method (from f2)
3319   // r2: receiver
3320   // r3: flags
3321 
3322   // Special case of invokeinterface called for virtual method of
3323   // java.lang.Object.  See cpCacheOop.cpp for details.
3324   // This code isn't produced by javac, but could be produced by
3325   // another compliant java compiler.
3326   Label notMethod;
3327   __ tbz(r3, ConstantPoolCacheEntry::is_forced_virtual_shift, notMethod);
3328 
3329   invokevirtual_helper(rmethod, r2, r3);
3330   __ bind(notMethod);
3331 
3332   // Get receiver klass into r3 - also a null check
3333   __ restore_locals();
3334   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3335   __ load_klass(r3, r2);
3336 
3337   Label no_such_interface, no_such_method;
3338 
3339   // Receiver subtype check against REFC.
3340   // Superklass in r0. Subklass in r3. Blows rscratch2, r13.
3341   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3342                              r3, r0, noreg,
3343                              // outputs: scan temp. reg, scan temp. reg
3344                              rscratch2, r13,
3345                              no_such_interface,
3346                              /*return_method=*/false);
3347 
3348   // profile this call
3349   __ profile_virtual_call(r3, r13, r19);
3350 
3351   // Get declaring interface class from method, and itable index
3352   __ ldr(r0, Address(rmethod, Method::const_offset()));
3353   __ ldr(r0, Address(r0, ConstMethod::constants_offset()));
3354   __ ldr(r0, Address(r0, ConstantPool::pool_holder_offset_in_bytes()));
3355   __ ldrw(rmethod, Address(rmethod, Method::itable_index_offset()));
3356   __ subw(rmethod, rmethod, Method::itable_index_max);
3357   __ negw(rmethod, rmethod);
3358 
3359   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3360                              r3, r0, rmethod,
3361                              // outputs: method, scan temp. reg
3362                              rmethod, r13,
3363                              no_such_interface);
3364 
3365   // rmethod,: methodOop to call
3366   // r2: receiver
3367   // Check for abstract method error
3368   // Note: This should be done more efficiently via a throw_abstract_method_error
3369   //       interpreter entry point and a conditional jump to it in case of a null
3370   //       method.
3371   __ cbz(rmethod, no_such_method);
3372 
3373   __ profile_arguments_type(r3, rmethod, r13, true);
3374 
3375   // do the call
3376   // r2: receiver
3377   // rmethod,: methodOop
3378   __ jump_from_interpreted(rmethod, r3);
3379   __ should_not_reach_here();
3380 
3381   // exception handling code follows...
3382   // note: must restore interpreter registers to canonical
3383   //       state for exception handling to work correctly!
3384 
3385   __ bind(no_such_method);
3386   // throw exception
3387   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3388   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3389   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
3390   // the call_VM checks for exception, so we should never return here.
3391   __ should_not_reach_here();
3392 
3393   __ bind(no_such_interface);
3394   // throw exception
3395   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3396   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3397   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3398                    InterpreterRuntime::throw_IncompatibleClassChangeError));
3399   // the call_VM checks for exception, so we should never return here.
3400   __ should_not_reach_here();
3401   return;
3402 }
3403 
3404 void TemplateTable::invokehandle(int byte_no) {
3405   transition(vtos, vtos);
3406   assert(byte_no == f1_byte, "use this argument");
3407 
3408   if (!EnableInvokeDynamic) {
3409     // rewriter does not generate this bytecode
3410     __ should_not_reach_here();
3411     return;
3412   }
3413 
3414   prepare_invoke(byte_no, rmethod, r0, r2);
3415   __ verify_method_ptr(r2);
3416   __ verify_oop(r2);
3417   __ null_check(r2);
3418 
3419   // FIXME: profile the LambdaForm also
3420 
3421   // r13 is safe to use here as a scratch reg because it is about to
3422   // be clobbered by jump_from_interpreted().
3423   __ profile_final_call(r13);
3424   __ profile_arguments_type(r13, rmethod, r4, true);
3425 
3426   __ jump_from_interpreted(rmethod, r0);
3427 }
3428 
3429 void TemplateTable::invokedynamic(int byte_no) {
3430   transition(vtos, vtos);
3431   assert(byte_no == f1_byte, "use this argument");
3432 
3433   if (!EnableInvokeDynamic) {
3434     // We should not encounter this bytecode if !EnableInvokeDynamic.
3435     // The verifier will stop it.  However, if we get past the verifier,
3436     // this will stop the thread in a reasonable way, without crashing the JVM.
3437     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3438                      InterpreterRuntime::throw_IncompatibleClassChangeError));
3439     // the call_VM checks for exception, so we should never return here.
3440     __ should_not_reach_here();
3441     return;
3442   }
3443 
3444   prepare_invoke(byte_no, rmethod, r0);
3445 
3446   // r0: CallSite object (from cpool->resolved_references[])
3447   // rmethod: MH.linkToCallSite method (from f2)
3448 
3449   // Note:  r0_callsite is already pushed by prepare_invoke
3450 
3451   // %%% should make a type profile for any invokedynamic that takes a ref argument
3452   // profile this call
3453   __ profile_call(rbcp);
3454   __ profile_arguments_type(r3, rmethod, r13, false);
3455 
3456   __ verify_oop(r0);
3457 
3458   __ jump_from_interpreted(rmethod, r0);
3459 }
3460 
3461 
3462 //-----------------------------------------------------------------------------
3463 // Allocation
3464 
3465 void TemplateTable::_new() {
3466   transition(vtos, atos);
3467 
3468   __ get_unsigned_2_byte_index_at_bcp(r3, 1);
3469   Label slow_case;
3470   Label done;
3471   Label initialize_header;
3472   Label initialize_object; // including clearing the fields
3473   Label allocate_shared;
3474 
3475   __ get_cpool_and_tags(r4, r0);
3476   // Make sure the class we're about to instantiate has been resolved.
3477   // This is done before loading InstanceKlass to be consistent with the order
3478   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3479   const int tags_offset = Array<u1>::base_offset_in_bytes();
3480   __ lea(rscratch1, Address(r0, r3, Address::lsl(0)));
3481   __ lea(rscratch1, Address(rscratch1, tags_offset));
3482   __ ldarb(rscratch1, rscratch1);
3483   __ cmp(rscratch1, JVM_CONSTANT_Class);
3484   __ br(Assembler::NE, slow_case);
3485 
3486   // get InstanceKlass
3487   __ lea(r4, Address(r4, r3, Address::lsl(3)));
3488   __ ldr(r4, Address(r4, sizeof(ConstantPool)));
3489 
3490   // make sure klass is initialized & doesn't have finalizer
3491   // make sure klass is fully initialized
3492   __ ldrb(rscratch1, Address(r4, InstanceKlass::init_state_offset()));
3493   __ cmp(rscratch1, InstanceKlass::fully_initialized);
3494   __ br(Assembler::NE, slow_case);
3495 
3496   // get instance_size in InstanceKlass (scaled to a count of bytes)
3497   __ ldrw(r3,
3498           Address(r4,
3499                   Klass::layout_helper_offset()));
3500   // test to see if it has a finalizer or is malformed in some way
3501   __ tbnz(r3, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
3502 
3503   // Allocate the instance
3504   // 1) Try to allocate in the TLAB
3505   // 2) if fail and the object is large allocate in the shared Eden
3506   // 3) if the above fails (or is not applicable), go to a slow case
3507   // (creates a new TLAB, etc.)
3508 
3509   const bool allow_shared_alloc =
3510     Universe::heap()->supports_inline_contig_alloc() && !CMSIncrementalMode;
3511 
3512   if (UseTLAB) {
3513     __ tlab_allocate(r0, r3, 0, noreg, r1,
3514                      allow_shared_alloc ? allocate_shared : slow_case);
3515 
3516     if (ZeroTLAB) {
3517       // the fields have been already cleared
3518       __ b(initialize_header);
3519     } else {
3520       // initialize both the header and fields
3521       __ b(initialize_object);
3522     }
3523   }
3524 
3525   // Allocation in the shared Eden, if allowed.
3526   //
3527   // r3: instance size in bytes
3528   if (allow_shared_alloc) {
3529     __ bind(allocate_shared);
3530 
3531     __ eden_allocate(r0, r3, 0, r10, slow_case);
3532     __ incr_allocated_bytes(rthread, r3, 0, rscratch1);
3533   }
3534 
3535   if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) {
3536     // The object is initialized before the header.  If the object size is
3537     // zero, go directly to the header initialization.
3538     __ bind(initialize_object);
3539     __ sub(r3, r3, sizeof(oopDesc));
3540     __ cbz(r3, initialize_header);
3541 
3542     // Initialize object fields
3543     {
3544       __ add(r2, r0, sizeof(oopDesc));
3545       Label loop;
3546       __ bind(loop);
3547       __ str(zr, Address(__ post(r2, BytesPerLong)));
3548       __ sub(r3, r3, BytesPerLong);
3549       __ cbnz(r3, loop);
3550     }
3551 
3552     // initialize object header only.
3553     __ bind(initialize_header);
3554     if (UseBiasedLocking) {
3555       __ ldr(rscratch1, Address(r4, Klass::prototype_header_offset()));
3556     } else {
3557       __ mov(rscratch1, (intptr_t)markOopDesc::prototype());
3558     }
3559     __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
3560     __ store_klass_gap(r0, zr);  // zero klass gap for compressed oops
3561     __ store_klass(r0, r4);      // store klass last
3562 
3563     {
3564       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3565       // Trigger dtrace event for fastpath
3566       __ push(atos); // save the return value
3567       __ call_VM_leaf(
3568            CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), r0);
3569       __ pop(atos); // restore the return value
3570 
3571     }
3572     __ b(done);
3573   }
3574 
3575   // slow case
3576   __ bind(slow_case);
3577   __ get_constant_pool(c_rarg1);
3578   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3579   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3580   __ verify_oop(r0);
3581 
3582   // continue
3583   __ bind(done);
3584   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3585   __ membar(Assembler::StoreStore);
3586 }
3587 
3588 void TemplateTable::newarray() {
3589   transition(itos, atos);
3590   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3591   __ mov(c_rarg2, r0);
3592   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3593           c_rarg1, c_rarg2);
3594   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3595   __ membar(Assembler::StoreStore);
3596 }
3597 
3598 void TemplateTable::anewarray() {
3599   transition(itos, atos);
3600   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3601   __ get_constant_pool(c_rarg1);
3602   __ mov(c_rarg3, r0);
3603   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3604           c_rarg1, c_rarg2, c_rarg3);
3605   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3606   __ membar(Assembler::StoreStore);
3607 }
3608 
3609 void TemplateTable::arraylength() {
3610   transition(atos, itos);
3611   __ null_check(r0, arrayOopDesc::length_offset_in_bytes());
3612   __ ldrw(r0, Address(r0, arrayOopDesc::length_offset_in_bytes()));
3613 }
3614 
3615 void TemplateTable::checkcast()
3616 {
3617   transition(atos, atos);
3618   Label done, is_null, ok_is_subtype, quicked, resolved;
3619   __ cbz(r0, is_null);
3620 
3621   // Get cpool & tags index
3622   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3623   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3624   // See if bytecode has already been quicked
3625   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3626   __ lea(r1, Address(rscratch1, r19));
3627   __ ldarb(r1, r1);
3628   __ cmp(r1, JVM_CONSTANT_Class);
3629   __ br(Assembler::EQ, quicked);
3630 
3631   __ push(atos); // save receiver for result, and for GC
3632   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3633   // vm_result_2 has metadata result
3634   __ get_vm_result_2(r0, rthread);
3635   __ pop(r3); // restore receiver
3636   __ b(resolved);
3637 
3638   // Get superklass in r0 and subklass in r3
3639   __ bind(quicked);
3640   __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
3641   __ lea(r0, Address(r2, r19, Address::lsl(3)));
3642   __ ldr(r0, Address(r0, sizeof(ConstantPool)));
3643 
3644   __ bind(resolved);
3645   __ load_klass(r19, r3);
3646 
3647   // Generate subtype check.  Blows r2, r5.  Object in r3.
3648   // Superklass in r0.  Subklass in r19.
3649   __ gen_subtype_check(r19, ok_is_subtype);
3650 
3651   // Come here on failure
3652   __ push(r3);
3653   // object is at TOS
3654   __ b(Interpreter::_throw_ClassCastException_entry);
3655 
3656   // Come here on success
3657   __ bind(ok_is_subtype);
3658   __ mov(r0, r3); // Restore object in r3
3659 
3660   // Collect counts on whether this test sees NULLs a lot or not.
3661   if (ProfileInterpreter) {
3662     __ b(done);
3663     __ bind(is_null);
3664     __ profile_null_seen(r2);
3665   } else {
3666     __ bind(is_null);   // same as 'done'
3667   }
3668   __ bind(done);
3669 }
3670 
3671 void TemplateTable::instanceof() {
3672   transition(atos, itos);
3673   Label done, is_null, ok_is_subtype, quicked, resolved;
3674   __ cbz(r0, is_null);
3675 
3676   // Get cpool & tags index
3677   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3678   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3679   // See if bytecode has already been quicked
3680   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3681   __ lea(r1, Address(rscratch1, r19));
3682   __ ldarb(r1, r1);
3683   __ cmp(r1, JVM_CONSTANT_Class);
3684   __ br(Assembler::EQ, quicked);
3685 
3686   __ push(atos); // save receiver for result, and for GC
3687   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3688   // vm_result_2 has metadata result
3689   __ get_vm_result_2(r0, rthread);
3690   __ pop(r3); // restore receiver
3691   __ verify_oop(r3);
3692   __ load_klass(r3, r3);
3693   __ b(resolved);
3694 
3695   // Get superklass in r0 and subklass in r3
3696   __ bind(quicked);
3697   __ load_klass(r3, r0);
3698   __ lea(r0, Address(r2, r19, Address::lsl(3)));
3699   __ ldr(r0, Address(r0, sizeof(ConstantPool)));
3700 
3701   __ bind(resolved);
3702 
3703   // Generate subtype check.  Blows r2, r5
3704   // Superklass in r0.  Subklass in r3.
3705   __ gen_subtype_check(r3, ok_is_subtype);
3706 
3707   // Come here on failure
3708   __ mov(r0, 0);
3709   __ b(done);
3710   // Come here on success
3711   __ bind(ok_is_subtype);
3712   __ mov(r0, 1);
3713 
3714   // Collect counts on whether this test sees NULLs a lot or not.
3715   if (ProfileInterpreter) {
3716     __ b(done);
3717     __ bind(is_null);
3718     __ profile_null_seen(r2);
3719   } else {
3720     __ bind(is_null);   // same as 'done'
3721   }
3722   __ bind(done);
3723   // r0 = 0: obj == NULL or  obj is not an instanceof the specified klass
3724   // r0 = 1: obj != NULL and obj is     an instanceof the specified klass
3725 }
3726 
3727 //-----------------------------------------------------------------------------
3728 // Breakpoints
3729 void TemplateTable::_breakpoint() {
3730   // Note: We get here even if we are single stepping..
3731   // jbug inists on setting breakpoints at every bytecode
3732   // even if we are in single step mode.
3733 
3734   transition(vtos, vtos);
3735 
3736   // get the unpatched byte code
3737   __ get_method(c_rarg1);
3738   __ call_VM(noreg,
3739              CAST_FROM_FN_PTR(address,
3740                               InterpreterRuntime::get_original_bytecode_at),
3741              c_rarg1, rbcp);
3742   __ mov(r19, r0);
3743 
3744   // post the breakpoint event
3745   __ call_VM(noreg,
3746              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
3747              rmethod, rbcp);
3748 
3749   // complete the execution of original bytecode
3750   __ mov(rscratch1, r19);
3751   __ dispatch_only_normal(vtos);
3752 }
3753 
3754 //-----------------------------------------------------------------------------
3755 // Exceptions
3756 
3757 void TemplateTable::athrow() {
3758   transition(atos, vtos);
3759   __ null_check(r0);
3760   __ b(Interpreter::throw_exception_entry());
3761 }
3762 
3763 //-----------------------------------------------------------------------------
3764 // Synchronization
3765 //
3766 // Note: monitorenter & exit are symmetric routines; which is reflected
3767 //       in the assembly code structure as well
3768 //
3769 // Stack layout:
3770 //
3771 // [expressions  ] <--- esp               = expression stack top
3772 // ..
3773 // [expressions  ]
3774 // [monitor entry] <--- monitor block top = expression stack bot
3775 // ..
3776 // [monitor entry]
3777 // [frame data   ] <--- monitor block bot
3778 // ...
3779 // [saved rbp    ] <--- rbp
3780 void TemplateTable::monitorenter()
3781 {
3782   transition(atos, vtos);
3783 
3784   // check for NULL object
3785   __ null_check(r0);
3786 
3787   const Address monitor_block_top(
3788         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3789   const Address monitor_block_bot(
3790         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3791   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3792 
3793   Label allocated;
3794 
3795   // initialize entry pointer
3796   __ mov(c_rarg1, zr); // points to free slot or NULL
3797 
3798   // find a free slot in the monitor block (result in c_rarg1)
3799   {
3800     Label entry, loop, exit;
3801     __ ldr(c_rarg3, monitor_block_top); // points to current entry,
3802                                         // starting with top-most entry
3803     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3804 
3805     __ b(entry);
3806 
3807     __ bind(loop);
3808     // check if current entry is used
3809     // if not used then remember entry in c_rarg1
3810     __ ldr(rscratch1, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
3811     __ cmp(zr, rscratch1);
3812     __ csel(c_rarg1, c_rarg3, c_rarg1, Assembler::EQ);
3813     // check if current entry is for same object
3814     __ cmp(r0, rscratch1);
3815     // if same object then stop searching
3816     __ br(Assembler::EQ, exit);
3817     // otherwise advance to next entry
3818     __ add(c_rarg3, c_rarg3, entry_size);
3819     __ bind(entry);
3820     // check if bottom reached
3821     __ cmp(c_rarg3, c_rarg2);
3822     // if not at bottom then check this entry
3823     __ br(Assembler::NE, loop);
3824     __ bind(exit);
3825   }
3826 
3827   __ cbnz(c_rarg1, allocated); // check if a slot has been found and
3828                             // if found, continue with that on
3829 
3830   // allocate one if there's no free slot
3831   {
3832     Label entry, loop;
3833     // 1. compute new pointers            // rsp: old expression stack top
3834     __ ldr(c_rarg1, monitor_block_bot);   // c_rarg1: old expression stack bottom
3835     __ sub(esp, esp, entry_size);         // move expression stack top
3836     __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
3837     __ mov(c_rarg3, esp);                 // set start value for copy loop
3838     __ str(c_rarg1, monitor_block_bot);   // set new monitor block bottom
3839 
3840     __ sub(sp, sp, entry_size);           // make room for the monitor
3841 
3842     __ b(entry);
3843     // 2. move expression stack contents
3844     __ bind(loop);
3845     __ ldr(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
3846                                                    // word from old location
3847     __ str(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
3848     __ add(c_rarg3, c_rarg3, wordSize);            // advance to next word
3849     __ bind(entry);
3850     __ cmp(c_rarg3, c_rarg1);        // check if bottom reached
3851     __ br(Assembler::NE, loop);      // if not at bottom then
3852                                      // copy next word
3853   }
3854 
3855   // call run-time routine
3856   // c_rarg1: points to monitor entry
3857   __ bind(allocated);
3858 
3859   // Increment bcp to point to the next bytecode, so exception
3860   // handling for async. exceptions work correctly.
3861   // The object has already been poped from the stack, so the
3862   // expression stack looks correct.
3863   __ increment(rbcp);
3864 
3865   // store object
3866   __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3867   __ lock_object(c_rarg1);
3868 
3869   // check to make sure this monitor doesn't cause stack overflow after locking
3870   __ save_bcp();  // in case of exception
3871   __ generate_stack_overflow_check(0);
3872 
3873   // The bcp has already been incremented. Just need to dispatch to
3874   // next instruction.
3875   __ dispatch_next(vtos);
3876 }
3877 
3878 
3879 void TemplateTable::monitorexit()
3880 {
3881   transition(atos, vtos);
3882 
3883   // check for NULL object
3884   __ null_check(r0);
3885 
3886   const Address monitor_block_top(
3887         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3888   const Address monitor_block_bot(
3889         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3890   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3891 
3892   Label found;
3893 
3894   // find matching slot
3895   {
3896     Label entry, loop;
3897     __ ldr(c_rarg1, monitor_block_top); // points to current entry,
3898                                         // starting with top-most entry
3899     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3900                                         // of monitor block
3901     __ b(entry);
3902 
3903     __ bind(loop);
3904     // check if current entry is for same object
3905     __ ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3906     __ cmp(r0, rscratch1);
3907     // if same object then stop searching
3908     __ br(Assembler::EQ, found);
3909     // otherwise advance to next entry
3910     __ add(c_rarg1, c_rarg1, entry_size);
3911     __ bind(entry);
3912     // check if bottom reached
3913     __ cmp(c_rarg1, c_rarg2);
3914     // if not at bottom then check this entry
3915     __ br(Assembler::NE, loop);
3916   }
3917 
3918   // error handling. Unlocking was not block-structured
3919   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3920                    InterpreterRuntime::throw_illegal_monitor_state_exception));
3921   __ should_not_reach_here();
3922 
3923   // call run-time routine
3924   __ bind(found);
3925   __ push_ptr(r0); // make sure object is on stack (contract with oopMaps)
3926   __ unlock_object(c_rarg1);
3927   __ pop_ptr(r0); // discard object
3928 }
3929 
3930 
3931 // Wide instructions
3932 void TemplateTable::wide()
3933 {
3934   __ load_unsigned_byte(r19, at_bcp(1));
3935   __ mov(rscratch1, (address)Interpreter::_wentry_point);
3936   __ ldr(rscratch1, Address(rscratch1, r19, Address::uxtw(3)));
3937   __ br(rscratch1);
3938 }
3939 
3940 
3941 // Multi arrays
3942 void TemplateTable::multianewarray() {
3943   transition(vtos, atos);
3944   __ load_unsigned_byte(r0, at_bcp(3)); // get number of dimensions
3945   // last dim is on top of stack; we want address of first one:
3946   // first_addr = last_addr + (ndims - 1) * wordSize
3947   __ lea(c_rarg1, Address(esp, r0, Address::uxtw(3)));
3948   __ sub(c_rarg1, c_rarg1, wordSize);
3949   call_VM(r0,
3950           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
3951           c_rarg1);
3952   __ load_unsigned_byte(r1, at_bcp(3));
3953   __ lea(esp, Address(esp, r1, Address::uxtw(3)));
3954 }
3955 #endif // !CC_INTERP
--- EOF ---