1 /*
   2  * Copyright (c) 2013, Red Hat Inc.
   3  * Copyright (c) 2003, 2011, Oracle and/or its affiliates.
   4  * All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "interpreter/interpreter.hpp"
  30 #include "interpreter/interpreterRuntime.hpp"
  31 #include "interpreter/templateTable.hpp"
  32 #include "memory/universe.inline.hpp"
  33 #include "oops/methodData.hpp"
  34 #include "oops/method.hpp"
  35 #include "oops/objArrayKlass.hpp"
  36 #include "oops/oop.inline.hpp"
  37 #include "prims/methodHandles.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/synchronizer.hpp"
  41 #if INCLUDE_ALL_GCS
  42 #include "shenandoahBarrierSetAssembler_aarch64.hpp"
  43 #endif
  44 
  45 #ifndef CC_INTERP
  46 
  47 #define __ _masm->
  48 
  49 // Platform-dependent initialization
  50 
  51 void TemplateTable::pd_initialize() {
  52   // No amd64 specific initialization
  53 }
  54 
  55 // Address computation: local variables
  56 
  57 static inline Address iaddress(int n) {
  58   return Address(rlocals, Interpreter::local_offset_in_bytes(n));
  59 }
  60 
  61 static inline Address laddress(int n) {
  62   return iaddress(n + 1);
  63 }
  64 
  65 static inline Address faddress(int n) {
  66   return iaddress(n);
  67 }
  68 
  69 static inline Address daddress(int n) {
  70   return laddress(n);
  71 }
  72 
  73 static inline Address aaddress(int n) {
  74   return iaddress(n);
  75 }
  76 
  77 static inline Address iaddress(Register r) {
  78   return Address(rlocals, r, Address::lsl(3));
  79 }
  80 
  81 static inline Address laddress(Register r, Register scratch,
  82                                InterpreterMacroAssembler* _masm) {
  83   __ lea(scratch, Address(rlocals, r, Address::lsl(3)));
  84   return Address(scratch, Interpreter::local_offset_in_bytes(1));
  85 }
  86 
  87 static inline Address faddress(Register r) {
  88   return iaddress(r);
  89 }
  90 
  91 static inline Address daddress(Register r, Register scratch,
  92                                InterpreterMacroAssembler* _masm) {
  93   return laddress(r, scratch, _masm);
  94 }
  95 
  96 static inline Address aaddress(Register r) {
  97   return iaddress(r);
  98 }
  99 
 100 static inline Address at_rsp() {
 101   return Address(esp, 0);
 102 }
 103 
 104 // At top of Java expression stack which may be different than esp().  It
 105 // isn't for category 1 objects.
 106 static inline Address at_tos   () {
 107   return Address(esp,  Interpreter::expr_offset_in_bytes(0));
 108 }
 109 
 110 static inline Address at_tos_p1() {
 111   return Address(esp,  Interpreter::expr_offset_in_bytes(1));
 112 }
 113 
 114 static inline Address at_tos_p2() {
 115   return Address(esp,  Interpreter::expr_offset_in_bytes(2));
 116 }
 117 
 118 static inline Address at_tos_p3() {
 119   return Address(esp,  Interpreter::expr_offset_in_bytes(3));
 120 }
 121 
 122 static inline Address at_tos_p4() {
 123   return Address(esp,  Interpreter::expr_offset_in_bytes(4));
 124 }
 125 
 126 static inline Address at_tos_p5() {
 127   return Address(esp,  Interpreter::expr_offset_in_bytes(5));
 128 }
 129 
 130 // Condition conversion
 131 static Assembler::Condition j_not(TemplateTable::Condition cc) {
 132   switch (cc) {
 133   case TemplateTable::equal        : return Assembler::NE;
 134   case TemplateTable::not_equal    : return Assembler::EQ;
 135   case TemplateTable::less         : return Assembler::GE;
 136   case TemplateTable::less_equal   : return Assembler::GT;
 137   case TemplateTable::greater      : return Assembler::LE;
 138   case TemplateTable::greater_equal: return Assembler::LT;
 139   }
 140   ShouldNotReachHere();
 141   return Assembler::EQ;
 142 }
 143 
 144 
 145 // Miscelaneous helper routines
 146 // Store an oop (or NULL) at the Address described by obj.
 147 // If val == noreg this means store a NULL
 148 static void do_oop_store(InterpreterMacroAssembler* _masm,
 149                          Address obj,
 150                          Register val,
 151                          BarrierSet::Name barrier,
 152                          bool precise) {
 153   assert(val == noreg || val == r0, "parameter is just for looks");
 154   switch (barrier) {
 155 #if INCLUDE_ALL_GCS
 156     case BarrierSet::G1SATBCT:
 157     case BarrierSet::G1SATBCTLogging:
 158       {
 159         // flatten object address if needed
 160         if (obj.index() == noreg && obj.offset() == 0) {
 161           if (obj.base() != r3) {
 162             __ mov(r3, obj.base());
 163           }
 164         } else {
 165           __ lea(r3, obj);
 166         }
 167         __ g1_write_barrier_pre(r3 /* obj */,
 168                                 r1 /* pre_val */,
 169                                 rthread /* thread */,
 170                                 r10  /* tmp */,
 171                                 val != noreg /* tosca_live */,
 172                                 false /* expand_call */);
 173         if (val == noreg) {
 174           __ store_heap_oop_null(Address(r3, 0));
 175         } else {
 176           // G1 barrier needs uncompressed oop for region cross check.
 177           Register new_val = val;
 178           if (UseCompressedOops) {
 179             new_val = rscratch2;
 180             __ mov(new_val, val);
 181           }
 182 
 183           __ store_heap_oop(Address(r3, 0), val);
 184 
 185           __ g1_write_barrier_post(r3 /* store_adr */,
 186                                    new_val /* new_val */,
 187                                    rthread /* thread */,
 188                                    r10 /* tmp */,
 189                                    r1 /* tmp2 */);
 190         }
 191 
 192       }
 193       break;
 194     case BarrierSet::ShenandoahBarrierSet:
 195       {
 196         // flatten object address if needed
 197         if (obj.index() == noreg && obj.offset() == 0) {
 198           if (obj.base() != r3) {
 199             __ mov(r3, obj.base());
 200           }
 201         } else {
 202           __ lea(r3, obj);
 203         }
 204         if (ShenandoahSATBBarrier) {
 205           __ g1_write_barrier_pre(r3 /* obj */,
 206                                   r1 /* pre_val */,
 207                                   rthread /* thread */,
 208                                   r10  /* tmp */,
 209                                   val != noreg /* tosca_live */,
 210                                   false /* expand_call */);
 211         }
 212         if (val == noreg) {
 213           __ store_heap_oop_null(Address(r3, 0));
 214         } else {
 215           if (ShenandoahStoreValEnqueueBarrier) {
 216             ShenandoahBarrierSetAssembler::bsasm()->storeval_barrier(_masm, val, r10);
 217           }
 218           __ store_heap_oop(Address(r3, 0), val);
 219         }
 220 
 221       }
 222       break;
 223 #endif // INCLUDE_ALL_GCS
 224     case BarrierSet::CardTableModRef:
 225     case BarrierSet::CardTableExtension:
 226       {
 227         if (val == noreg) {
 228           __ store_heap_oop_null(obj);
 229         } else {
 230           __ store_heap_oop(obj, val);
 231           // flatten object address if needed
 232           if (!precise || (obj.index() == noreg && obj.offset() == 0)) {
 233             __ store_check(obj.base());
 234           } else {
 235             __ lea(r3, obj);
 236             __ store_check(r3);
 237           }
 238         }
 239       }
 240       break;
 241     case BarrierSet::ModRef:
 242     case BarrierSet::Other:
 243       if (val == noreg) {
 244         __ store_heap_oop_null(obj);
 245       } else {
 246         __ store_heap_oop(obj, val);
 247       }
 248       break;
 249     default      :
 250       ShouldNotReachHere();
 251 
 252   }
 253 }
 254 
 255 Address TemplateTable::at_bcp(int offset) {
 256   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 257   return Address(rbcp, offset);
 258 }
 259 
 260 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 261                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 262                                    int byte_no)
 263 {
 264   if (!RewriteBytecodes)  return;
 265   Label L_patch_done;
 266 
 267   switch (bc) {
 268   case Bytecodes::_fast_aputfield:
 269   case Bytecodes::_fast_bputfield:
 270   case Bytecodes::_fast_zputfield:
 271   case Bytecodes::_fast_cputfield:
 272   case Bytecodes::_fast_dputfield:
 273   case Bytecodes::_fast_fputfield:
 274   case Bytecodes::_fast_iputfield:
 275   case Bytecodes::_fast_lputfield:
 276   case Bytecodes::_fast_sputfield:
 277     {
 278       // We skip bytecode quickening for putfield instructions when
 279       // the put_code written to the constant pool cache is zero.
 280       // This is required so that every execution of this instruction
 281       // calls out to InterpreterRuntime::resolve_get_put to do
 282       // additional, required work.
 283       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 284       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 285       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 286       __ movw(bc_reg, bc);
 287       __ cmpw(temp_reg, (unsigned) 0);
 288       __ br(Assembler::EQ, L_patch_done);  // don't patch
 289     }
 290     break;
 291   default:
 292     assert(byte_no == -1, "sanity");
 293     // the pair bytecodes have already done the load.
 294     if (load_bc_into_bc_reg) {
 295       __ movw(bc_reg, bc);
 296     }
 297   }
 298 
 299   if (JvmtiExport::can_post_breakpoint()) {
 300     Label L_fast_patch;
 301     // if a breakpoint is present we can't rewrite the stream directly
 302     __ load_unsigned_byte(temp_reg, at_bcp(0));
 303     __ cmpw(temp_reg, Bytecodes::_breakpoint);
 304     __ br(Assembler::NE, L_fast_patch);
 305     // Let breakpoint table handling rewrite to quicker bytecode
 306     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), rmethod, rbcp, bc_reg);
 307     __ b(L_patch_done);
 308     __ bind(L_fast_patch);
 309   }
 310 
 311 #ifdef ASSERT
 312   Label L_okay;
 313   __ load_unsigned_byte(temp_reg, at_bcp(0));
 314   __ cmpw(temp_reg, (int) Bytecodes::java_code(bc));
 315   __ br(Assembler::EQ, L_okay);
 316   __ cmpw(temp_reg, bc_reg);
 317   __ br(Assembler::EQ, L_okay);
 318   __ stop("patching the wrong bytecode");
 319   __ bind(L_okay);
 320 #endif
 321 
 322   // patch bytecode
 323   __ strb(bc_reg, at_bcp(0));
 324   __ bind(L_patch_done);
 325 }
 326 
 327 
 328 // Individual instructions
 329 
 330 void TemplateTable::nop() {
 331   transition(vtos, vtos);
 332   // nothing to do
 333 }
 334 
 335 void TemplateTable::shouldnotreachhere() {
 336   transition(vtos, vtos);
 337   __ stop("shouldnotreachhere bytecode");
 338 }
 339 
 340 void TemplateTable::aconst_null()
 341 {
 342   transition(vtos, atos);
 343   __ mov(r0, 0);
 344 }
 345 
 346 void TemplateTable::iconst(int value)
 347 {
 348   transition(vtos, itos);
 349   __ mov(r0, value);
 350 }
 351 
 352 void TemplateTable::lconst(int value)
 353 {
 354   __ mov(r0, value);
 355 }
 356 
 357 void TemplateTable::fconst(int value)
 358 {
 359   transition(vtos, ftos);
 360   switch (value) {
 361   case 0:
 362     __ fmovs(v0, zr);
 363     break;
 364   case 1:
 365     __ fmovs(v0, 1.0);
 366     break;
 367   case 2:
 368     __ fmovs(v0, 2.0);
 369     break;
 370   default:
 371     ShouldNotReachHere();
 372     break;
 373   }
 374 }
 375 
 376 void TemplateTable::dconst(int value)
 377 {
 378   transition(vtos, dtos);
 379   switch (value) {
 380   case 0:
 381     __ fmovd(v0, zr);
 382     break;
 383   case 1:
 384     __ fmovd(v0, 1.0);
 385     break;
 386   case 2:
 387     __ fmovd(v0, 2.0);
 388     break;
 389   default:
 390     ShouldNotReachHere();
 391     break;
 392   }
 393 }
 394 
 395 void TemplateTable::bipush()
 396 {
 397   transition(vtos, itos);
 398   __ load_signed_byte32(r0, at_bcp(1));
 399 }
 400 
 401 void TemplateTable::sipush()
 402 {
 403   transition(vtos, itos);
 404   __ load_unsigned_short(r0, at_bcp(1));
 405   __ revw(r0, r0);
 406   __ asrw(r0, r0, 16);
 407 }
 408 
 409 void TemplateTable::ldc(bool wide)
 410 {
 411   transition(vtos, vtos);
 412   Label call_ldc, notFloat, notClass, Done;
 413 
 414   if (wide) {
 415     __ get_unsigned_2_byte_index_at_bcp(r1, 1);
 416   } else {
 417     __ load_unsigned_byte(r1, at_bcp(1));
 418   }
 419   __ get_cpool_and_tags(r2, r0);
 420 
 421   const int base_offset = ConstantPool::header_size() * wordSize;
 422   const int tags_offset = Array<u1>::base_offset_in_bytes();
 423 
 424   // get type
 425   __ add(r3, r1, tags_offset);
 426   __ lea(r3, Address(r0, r3));
 427   __ ldarb(r3, r3);
 428 
 429   // unresolved class - get the resolved class
 430   __ cmp(r3, JVM_CONSTANT_UnresolvedClass);
 431   __ br(Assembler::EQ, call_ldc);
 432 
 433   // unresolved class in error state - call into runtime to throw the error
 434   // from the first resolution attempt
 435   __ cmp(r3, JVM_CONSTANT_UnresolvedClassInError);
 436   __ br(Assembler::EQ, call_ldc);
 437 
 438   // resolved class - need to call vm to get java mirror of the class
 439   __ cmp(r3, JVM_CONSTANT_Class);
 440   __ br(Assembler::NE, notClass);
 441 
 442   __ bind(call_ldc);
 443   __ mov(c_rarg1, wide);
 444   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 445   __ push_ptr(r0);
 446   __ verify_oop(r0);
 447   __ b(Done);
 448 
 449   __ bind(notClass);
 450   __ cmp(r3, JVM_CONSTANT_Float);
 451   __ br(Assembler::NE, notFloat);
 452   // ftos
 453   __ adds(r1, r2, r1, Assembler::LSL, 3);
 454   __ ldrs(v0, Address(r1, base_offset));
 455   __ push_f();
 456   __ b(Done);
 457 
 458   __ bind(notFloat);
 459 #ifdef ASSERT
 460   {
 461     Label L;
 462     __ cmp(r3, JVM_CONSTANT_Integer);
 463     __ br(Assembler::EQ, L);
 464     // String and Object are rewritten to fast_aldc
 465     __ stop("unexpected tag type in ldc");
 466     __ bind(L);
 467   }
 468 #endif
 469   // itos JVM_CONSTANT_Integer only
 470   __ adds(r1, r2, r1, Assembler::LSL, 3);
 471   __ ldrw(r0, Address(r1, base_offset));
 472   __ push_i(r0);
 473   __ bind(Done);
 474 }
 475 
 476 // Fast path for caching oop constants.
 477 void TemplateTable::fast_aldc(bool wide)
 478 {
 479   transition(vtos, atos);
 480 
 481   Register result = r0;
 482   Register tmp = r1;
 483   int index_size = wide ? sizeof(u2) : sizeof(u1);
 484 
 485   Label resolved;
 486 
 487   // We are resolved if the resolved reference cache entry contains a
 488   // non-null object (String, MethodType, etc.)
 489   assert_different_registers(result, tmp);
 490   __ get_cache_index_at_bcp(tmp, 1, index_size);
 491   __ load_resolved_reference_at_index(result, tmp);
 492   __ cbnz(result, resolved);
 493 
 494   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 495 
 496   // first time invocation - must resolve first
 497   __ mov(tmp, (int)bytecode());
 498   __ call_VM(result, entry, tmp);
 499 
 500   __ bind(resolved);
 501 
 502   if (VerifyOops) {
 503     __ verify_oop(result);
 504   }
 505 }
 506 
 507 void TemplateTable::ldc2_w()
 508 {
 509   transition(vtos, vtos);
 510   Label Long, Done;
 511   __ get_unsigned_2_byte_index_at_bcp(r0, 1);
 512 
 513   __ get_cpool_and_tags(r1, r2);
 514   const int base_offset = ConstantPool::header_size() * wordSize;
 515   const int tags_offset = Array<u1>::base_offset_in_bytes();
 516 
 517   // get type
 518   __ lea(r2, Address(r2, r0, Address::lsl(0)));
 519   __ load_unsigned_byte(r2, Address(r2, tags_offset));
 520   __ cmpw(r2, (int)JVM_CONSTANT_Double);
 521   __ br(Assembler::NE, Long);
 522   // dtos
 523   __ lea (r2, Address(r1, r0, Address::lsl(3)));
 524   __ ldrd(v0, Address(r2, base_offset));
 525   __ push_d();
 526   __ b(Done);
 527 
 528   __ bind(Long);
 529   // ltos
 530   __ lea(r0, Address(r1, r0, Address::lsl(3)));
 531   __ ldr(r0, Address(r0, base_offset));
 532   __ push_l();
 533 
 534   __ bind(Done);
 535 }
 536 
 537 void TemplateTable::locals_index(Register reg, int offset)
 538 {
 539   __ ldrb(reg, at_bcp(offset));
 540   __ neg(reg, reg);
 541 }
 542 
 543 void TemplateTable::iload()
 544 {
 545   transition(vtos, itos);
 546   if (RewriteFrequentPairs) {
 547     Label rewrite, done;
 548     Register bc = r4;
 549 
 550     // get next bytecode
 551     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 552 
 553     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 554     // last two iloads in a pair.  Comparing against fast_iload means that
 555     // the next bytecode is neither an iload or a caload, and therefore
 556     // an iload pair.
 557     __ cmpw(r1, Bytecodes::_iload);
 558     __ br(Assembler::EQ, done);
 559 
 560     // if _fast_iload rewrite to _fast_iload2
 561     __ cmpw(r1, Bytecodes::_fast_iload);
 562     __ movw(bc, Bytecodes::_fast_iload2);
 563     __ br(Assembler::EQ, rewrite);
 564 
 565     // if _caload rewrite to _fast_icaload
 566     __ cmpw(r1, Bytecodes::_caload);
 567     __ movw(bc, Bytecodes::_fast_icaload);
 568     __ br(Assembler::EQ, rewrite);
 569 
 570     // else rewrite to _fast_iload
 571     __ movw(bc, Bytecodes::_fast_iload);
 572 
 573     // rewrite
 574     // bc: new bytecode
 575     __ bind(rewrite);
 576     patch_bytecode(Bytecodes::_iload, bc, r1, false);
 577     __ bind(done);
 578 
 579   }
 580 
 581   // do iload, get the local value into tos
 582   locals_index(r1);
 583   __ ldr(r0, iaddress(r1));
 584 
 585 }
 586 
 587 void TemplateTable::fast_iload2()
 588 {
 589   transition(vtos, itos);
 590   locals_index(r1);
 591   __ ldr(r0, iaddress(r1));
 592   __ push(itos);
 593   locals_index(r1, 3);
 594   __ ldr(r0, iaddress(r1));
 595 }
 596 
 597 void TemplateTable::fast_iload()
 598 {
 599   transition(vtos, itos);
 600   locals_index(r1);
 601   __ ldr(r0, iaddress(r1));
 602 }
 603 
 604 void TemplateTable::lload()
 605 {
 606   transition(vtos, ltos);
 607   __ ldrb(r1, at_bcp(1));
 608   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 609   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 610 }
 611 
 612 void TemplateTable::fload()
 613 {
 614   transition(vtos, ftos);
 615   locals_index(r1);
 616   // n.b. we use ldrd here because this is a 64 bit slot
 617   // this is comparable to the iload case
 618   __ ldrd(v0, faddress(r1));
 619 }
 620 
 621 void TemplateTable::dload()
 622 {
 623   transition(vtos, dtos);
 624   __ ldrb(r1, at_bcp(1));
 625   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 626   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 627 }
 628 
 629 void TemplateTable::aload()
 630 {
 631   transition(vtos, atos);
 632   locals_index(r1);
 633   __ ldr(r0, iaddress(r1));
 634 }
 635 
 636 void TemplateTable::locals_index_wide(Register reg) {
 637   __ ldrh(reg, at_bcp(2));
 638   __ rev16w(reg, reg);
 639   __ neg(reg, reg);
 640 }
 641 
 642 void TemplateTable::wide_iload() {
 643   transition(vtos, itos);
 644   locals_index_wide(r1);
 645   __ ldr(r0, iaddress(r1));
 646 }
 647 
 648 void TemplateTable::wide_lload()
 649 {
 650   transition(vtos, ltos);
 651   __ ldrh(r1, at_bcp(2));
 652   __ rev16w(r1, r1);
 653   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 654   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 655 }
 656 
 657 void TemplateTable::wide_fload()
 658 {
 659   transition(vtos, ftos);
 660   locals_index_wide(r1);
 661   // n.b. we use ldrd here because this is a 64 bit slot
 662   // this is comparable to the iload case
 663   __ ldrd(v0, faddress(r1));
 664 }
 665 
 666 void TemplateTable::wide_dload()
 667 {
 668   transition(vtos, dtos);
 669   __ ldrh(r1, at_bcp(2));
 670   __ rev16w(r1, r1);
 671   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 672   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 673 }
 674 
 675 void TemplateTable::wide_aload()
 676 {
 677   transition(vtos, atos);
 678   locals_index_wide(r1);
 679   __ ldr(r0, aaddress(r1));
 680 }
 681 
 682 void TemplateTable::index_check(Register array, Register index)
 683 {
 684   // destroys r1, rscratch1
 685   // check array
 686   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
 687   // sign extend index for use by indexed load
 688   // __ movl2ptr(index, index);
 689   // check index
 690   Register length = rscratch1;
 691   __ ldrw(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 692   __ cmpw(index, length);
 693   if (index != r1) {
 694     // ??? convention: move aberrant index into r1 for exception message
 695     assert(r1 != array, "different registers");
 696     __ mov(r1, index);
 697   }
 698   Label ok;
 699   __ br(Assembler::LO, ok);
 700   __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 701   __ br(rscratch1);
 702   __ bind(ok);
 703 }
 704 
 705 void TemplateTable::iaload()
 706 {
 707   transition(itos, itos);
 708   __ mov(r1, r0);
 709   __ pop_ptr(r0);
 710   // r0: array
 711   // r1: index
 712   index_check(r0, r1); // leaves index in r1, kills rscratch1
 713   __ lea(r1, Address(r0, r1, Address::uxtw(2)));
 714   __ ldrw(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_INT)));
 715 }
 716 
 717 void TemplateTable::laload()
 718 {
 719   transition(itos, ltos);
 720   __ mov(r1, r0);
 721   __ pop_ptr(r0);
 722   // r0: array
 723   // r1: index
 724   index_check(r0, r1); // leaves index in r1, kills rscratch1
 725   __ lea(r1, Address(r0, r1, Address::uxtw(3)));
 726   __ ldr(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_LONG)));
 727 }
 728 
 729 void TemplateTable::faload()
 730 {
 731   transition(itos, ftos);
 732   __ mov(r1, r0);
 733   __ pop_ptr(r0);
 734   // r0: array
 735   // r1: index
 736   index_check(r0, r1); // leaves index in r1, kills rscratch1
 737   __ lea(r1,  Address(r0, r1, Address::uxtw(2)));
 738   __ ldrs(v0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
 739 }
 740 
 741 void TemplateTable::daload()
 742 {
 743   transition(itos, dtos);
 744   __ mov(r1, r0);
 745   __ pop_ptr(r0);
 746   // r0: array
 747   // r1: index
 748   index_check(r0, r1); // leaves index in r1, kills rscratch1
 749   __ lea(r1,  Address(r0, r1, Address::uxtw(3)));
 750   __ ldrd(v0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
 751 }
 752 
 753 void TemplateTable::aaload()
 754 {
 755   transition(itos, atos);
 756   __ mov(r1, r0);
 757   __ pop_ptr(r0);
 758   // r0: array
 759   // r1: index
 760   index_check(r0, r1); // leaves index in r1, kills rscratch1
 761   int s = (UseCompressedOops ? 2 : 3);
 762   __ lea(r1, Address(r0, r1, Address::uxtw(s)));
 763   __ load_heap_oop(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
 764 }
 765 
 766 void TemplateTable::baload()
 767 {
 768   transition(itos, itos);
 769   __ mov(r1, r0);
 770   __ pop_ptr(r0);
 771   // r0: array
 772   // r1: index
 773   index_check(r0, r1); // leaves index in r1, kills rscratch1
 774   __ lea(r1,  Address(r0, r1, Address::uxtw(0)));
 775   __ load_signed_byte(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_BYTE)));
 776 }
 777 
 778 void TemplateTable::caload()
 779 {
 780   transition(itos, itos);
 781   __ mov(r1, r0);
 782   __ pop_ptr(r0);
 783   // r0: array
 784   // r1: index
 785   index_check(r0, r1); // leaves index in r1, kills rscratch1
 786   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 787   __ load_unsigned_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 788 }
 789 
 790 // iload followed by caload frequent pair
 791 void TemplateTable::fast_icaload()
 792 {
 793   transition(vtos, itos);
 794   // load index out of locals
 795   locals_index(r2);
 796   __ ldr(r1, iaddress(r2));
 797 
 798   __ pop_ptr(r0);
 799 
 800   // r0: array
 801   // r1: index
 802   index_check(r0, r1); // leaves index in r1, kills rscratch1
 803   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 804   __ load_unsigned_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 805 }
 806 
 807 void TemplateTable::saload()
 808 {
 809   transition(itos, itos);
 810   __ mov(r1, r0);
 811   __ pop_ptr(r0);
 812   // r0: array
 813   // r1: index
 814   index_check(r0, r1); // leaves index in r1, kills rscratch1
 815   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 816   __ load_signed_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_SHORT)));
 817 }
 818 
 819 void TemplateTable::iload(int n)
 820 {
 821   transition(vtos, itos);
 822   __ ldr(r0, iaddress(n));
 823 }
 824 
 825 void TemplateTable::lload(int n)
 826 {
 827   transition(vtos, ltos);
 828   __ ldr(r0, laddress(n));
 829 }
 830 
 831 void TemplateTable::fload(int n)
 832 {
 833   transition(vtos, ftos);
 834   __ ldrs(v0, faddress(n));
 835 }
 836 
 837 void TemplateTable::dload(int n)
 838 {
 839   transition(vtos, dtos);
 840   __ ldrd(v0, daddress(n));
 841 }
 842 
 843 void TemplateTable::aload(int n)
 844 {
 845   transition(vtos, atos);
 846   __ ldr(r0, iaddress(n));
 847 }
 848 
 849 void TemplateTable::aload_0()
 850 {
 851   // According to bytecode histograms, the pairs:
 852   //
 853   // _aload_0, _fast_igetfield
 854   // _aload_0, _fast_agetfield
 855   // _aload_0, _fast_fgetfield
 856   //
 857   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 858   // _aload_0 bytecode checks if the next bytecode is either
 859   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 860   // rewrites the current bytecode into a pair bytecode; otherwise it
 861   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 862   // the pair check anymore.
 863   //
 864   // Note: If the next bytecode is _getfield, the rewrite must be
 865   //       delayed, otherwise we may miss an opportunity for a pair.
 866   //
 867   // Also rewrite frequent pairs
 868   //   aload_0, aload_1
 869   //   aload_0, iload_1
 870   // These bytecodes with a small amount of code are most profitable
 871   // to rewrite
 872   if (RewriteFrequentPairs) {
 873     Label rewrite, done;
 874     const Register bc = r4;
 875 
 876     // get next bytecode
 877     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 878 
 879     // do actual aload_0
 880     aload(0);
 881 
 882     // if _getfield then wait with rewrite
 883     __ cmpw(r1, Bytecodes::Bytecodes::_getfield);
 884     __ br(Assembler::EQ, done);
 885 
 886     // if _igetfield then reqrite to _fast_iaccess_0
 887     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 888     __ cmpw(r1, Bytecodes::_fast_igetfield);
 889     __ movw(bc, Bytecodes::_fast_iaccess_0);
 890     __ br(Assembler::EQ, rewrite);
 891 
 892     // if _agetfield then reqrite to _fast_aaccess_0
 893     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 894     __ cmpw(r1, Bytecodes::_fast_agetfield);
 895     __ movw(bc, Bytecodes::_fast_aaccess_0);
 896     __ br(Assembler::EQ, rewrite);
 897 
 898     // if _fgetfield then reqrite to _fast_faccess_0
 899     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 900     __ cmpw(r1, Bytecodes::_fast_fgetfield);
 901     __ movw(bc, Bytecodes::_fast_faccess_0);
 902     __ br(Assembler::EQ, rewrite);
 903 
 904     // else rewrite to _fast_aload0
 905     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 906     __ movw(bc, Bytecodes::Bytecodes::_fast_aload_0);
 907 
 908     // rewrite
 909     // bc: new bytecode
 910     __ bind(rewrite);
 911     patch_bytecode(Bytecodes::_aload_0, bc, r1, false);
 912 
 913     __ bind(done);
 914   } else {
 915     aload(0);
 916   }
 917 }
 918 
 919 void TemplateTable::istore()
 920 {
 921   transition(itos, vtos);
 922   locals_index(r1);
 923   // FIXME: We're being very pernickerty here storing a jint in a
 924   // local with strw, which costs an extra instruction over what we'd
 925   // be able to do with a simple str.  We should just store the whole
 926   // word.
 927   __ lea(rscratch1, iaddress(r1));
 928   __ strw(r0, Address(rscratch1));
 929 }
 930 
 931 void TemplateTable::lstore()
 932 {
 933   transition(ltos, vtos);
 934   locals_index(r1);
 935   __ str(r0, laddress(r1, rscratch1, _masm));
 936 }
 937 
 938 void TemplateTable::fstore() {
 939   transition(ftos, vtos);
 940   locals_index(r1);
 941   __ lea(rscratch1, iaddress(r1));
 942   __ strs(v0, Address(rscratch1));
 943 }
 944 
 945 void TemplateTable::dstore() {
 946   transition(dtos, vtos);
 947   locals_index(r1);
 948   __ strd(v0, daddress(r1, rscratch1, _masm));
 949 }
 950 
 951 void TemplateTable::astore()
 952 {
 953   transition(vtos, vtos);
 954   __ pop_ptr(r0);
 955   locals_index(r1);
 956   __ str(r0, aaddress(r1));
 957 }
 958 
 959 void TemplateTable::wide_istore() {
 960   transition(vtos, vtos);
 961   __ pop_i();
 962   locals_index_wide(r1);
 963   __ lea(rscratch1, iaddress(r1));
 964   __ strw(r0, Address(rscratch1));
 965 }
 966 
 967 void TemplateTable::wide_lstore() {
 968   transition(vtos, vtos);
 969   __ pop_l();
 970   locals_index_wide(r1);
 971   __ str(r0, laddress(r1, rscratch1, _masm));
 972 }
 973 
 974 void TemplateTable::wide_fstore() {
 975   transition(vtos, vtos);
 976   __ pop_f();
 977   locals_index_wide(r1);
 978   __ lea(rscratch1, faddress(r1));
 979   __ strs(v0, rscratch1);
 980 }
 981 
 982 void TemplateTable::wide_dstore() {
 983   transition(vtos, vtos);
 984   __ pop_d();
 985   locals_index_wide(r1);
 986   __ strd(v0, daddress(r1, rscratch1, _masm));
 987 }
 988 
 989 void TemplateTable::wide_astore() {
 990   transition(vtos, vtos);
 991   __ pop_ptr(r0);
 992   locals_index_wide(r1);
 993   __ str(r0, aaddress(r1));
 994 }
 995 
 996 void TemplateTable::iastore() {
 997   transition(itos, vtos);
 998   __ pop_i(r1);
 999   __ pop_ptr(r3);
1000   // r0: value
1001   // r1: index
1002   // r3: array
1003   index_check(r3, r1); // prefer index in r1
1004   __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
1005   __ strw(r0, Address(rscratch1,
1006                       arrayOopDesc::base_offset_in_bytes(T_INT)));
1007 }
1008 
1009 void TemplateTable::lastore() {
1010   transition(ltos, vtos);
1011   __ pop_i(r1);
1012   __ pop_ptr(r3);
1013   // r0: value
1014   // r1: index
1015   // r3: array
1016   index_check(r3, r1); // prefer index in r1
1017   __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
1018   __ str(r0, Address(rscratch1,
1019                       arrayOopDesc::base_offset_in_bytes(T_LONG)));
1020 }
1021 
1022 void TemplateTable::fastore() {
1023   transition(ftos, vtos);
1024   __ pop_i(r1);
1025   __ pop_ptr(r3);
1026   // v0: value
1027   // r1:  index
1028   // r3:  array
1029   index_check(r3, r1); // prefer index in r1
1030   __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
1031   __ strs(v0, Address(rscratch1,
1032                       arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
1033 }
1034 
1035 void TemplateTable::dastore() {
1036   transition(dtos, vtos);
1037   __ pop_i(r1);
1038   __ pop_ptr(r3);
1039   // v0: value
1040   // r1:  index
1041   // r3:  array
1042   index_check(r3, r1); // prefer index in r1
1043   __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
1044   __ strd(v0, Address(rscratch1,
1045                       arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
1046 }
1047 
1048 void TemplateTable::aastore() {
1049   Label is_null, ok_is_subtype, done;
1050   transition(vtos, vtos);
1051   // stack: ..., array, index, value
1052   __ ldr(r0, at_tos());    // value
1053   __ ldr(r2, at_tos_p1()); // index
1054   __ ldr(r3, at_tos_p2()); // array
1055 
1056   Address element_address(r4, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
1057 
1058   index_check(r3, r2);     // kills r1
1059   __ lea(r4, Address(r3, r2, Address::uxtw(UseCompressedOops? 2 : 3)));
1060 
1061   // do array store check - check for NULL value first
1062   __ cbz(r0, is_null);
1063 
1064   // Move subklass into r1
1065   __ load_klass(r1, r0);
1066   // Move superklass into r0
1067   __ load_klass(r0, r3);
1068   __ ldr(r0, Address(r0,
1069                      ObjArrayKlass::element_klass_offset()));
1070   // Compress array + index*oopSize + 12 into a single register.  Frees r2.
1071 
1072   // Generate subtype check.  Blows r2, r5
1073   // Superklass in r0.  Subklass in r1.
1074   __ gen_subtype_check(r1, ok_is_subtype);
1075 
1076   // Come here on failure
1077   // object is at TOS
1078   __ b(Interpreter::_throw_ArrayStoreException_entry);
1079 
1080   // Come here on success
1081   __ bind(ok_is_subtype);
1082 
1083   // Get the value we will store
1084   __ ldr(r0, at_tos());
1085   // Now store using the appropriate barrier
1086   do_oop_store(_masm, element_address, r0, _bs->kind(), true);
1087   __ b(done);
1088 
1089   // Have a NULL in r0, r3=array, r2=index.  Store NULL at ary[idx]
1090   __ bind(is_null);
1091   __ profile_null_seen(r2);
1092 
1093   // Store a NULL
1094   do_oop_store(_masm, element_address, noreg, _bs->kind(), true);
1095 
1096   // Pop stack arguments
1097   __ bind(done);
1098   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1099 }
1100 
1101 void TemplateTable::bastore()
1102 {
1103   transition(itos, vtos);
1104   __ pop_i(r1);
1105   __ pop_ptr(r3);
1106   // r0: value
1107   // r1: index
1108   // r3: array
1109   index_check(r3, r1); // prefer index in r1
1110 
1111   // Need to check whether array is boolean or byte
1112   // since both types share the bastore bytecode.
1113   __ load_klass(r2, r3);
1114   __ ldrw(r2, Address(r2, Klass::layout_helper_offset()));
1115   int diffbit = Klass::layout_helper_boolean_diffbit();
1116   __ andw(rscratch1, r2, diffbit);
1117   Label L_skip;
1118   __ cbzw(rscratch1, L_skip);
1119   __ andw(r0, r0, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1120   __ bind(L_skip);
1121 
1122   __ lea(rscratch1, Address(r3, r1, Address::uxtw(0)));
1123   __ strb(r0, Address(rscratch1,
1124                       arrayOopDesc::base_offset_in_bytes(T_BYTE)));
1125 }
1126 
1127 void TemplateTable::castore()
1128 {
1129   transition(itos, vtos);
1130   __ pop_i(r1);
1131   __ pop_ptr(r3);
1132   // r0: value
1133   // r1: index
1134   // r3: array
1135   index_check(r3, r1); // prefer index in r1
1136   __ lea(rscratch1, Address(r3, r1, Address::uxtw(1)));
1137   __ strh(r0, Address(rscratch1,
1138                       arrayOopDesc::base_offset_in_bytes(T_CHAR)));
1139 }
1140 
1141 void TemplateTable::sastore()
1142 {
1143   castore();
1144 }
1145 
1146 void TemplateTable::istore(int n)
1147 {
1148   transition(itos, vtos);
1149   __ str(r0, iaddress(n));
1150 }
1151 
1152 void TemplateTable::lstore(int n)
1153 {
1154   transition(ltos, vtos);
1155   __ str(r0, laddress(n));
1156 }
1157 
1158 void TemplateTable::fstore(int n)
1159 {
1160   transition(ftos, vtos);
1161   __ strs(v0, faddress(n));
1162 }
1163 
1164 void TemplateTable::dstore(int n)
1165 {
1166   transition(dtos, vtos);
1167   __ strd(v0, daddress(n));
1168 }
1169 
1170 void TemplateTable::astore(int n)
1171 {
1172   transition(vtos, vtos);
1173   __ pop_ptr(r0);
1174   __ str(r0, iaddress(n));
1175 }
1176 
1177 void TemplateTable::pop()
1178 {
1179   transition(vtos, vtos);
1180   __ add(esp, esp, Interpreter::stackElementSize);
1181 }
1182 
1183 void TemplateTable::pop2()
1184 {
1185   transition(vtos, vtos);
1186   __ add(esp, esp, 2 * Interpreter::stackElementSize);
1187 }
1188 
1189 void TemplateTable::dup()
1190 {
1191   transition(vtos, vtos);
1192   __ ldr(r0, Address(esp, 0));
1193   __ push(r0);
1194   // stack: ..., a, a
1195 }
1196 
1197 void TemplateTable::dup_x1()
1198 {
1199   transition(vtos, vtos);
1200   // stack: ..., a, b
1201   __ ldr(r0, at_tos());  // load b
1202   __ ldr(r2, at_tos_p1());  // load a
1203   __ str(r0, at_tos_p1());  // store b
1204   __ str(r2, at_tos());  // store a
1205   __ push(r0);                  // push b
1206   // stack: ..., b, a, b
1207 }
1208 
1209 void TemplateTable::dup_x2()
1210 {
1211   transition(vtos, vtos);
1212   // stack: ..., a, b, c
1213   __ ldr(r0, at_tos());  // load c
1214   __ ldr(r2, at_tos_p2());  // load a
1215   __ str(r0, at_tos_p2());  // store c in a
1216   __ push(r0);      // push c
1217   // stack: ..., c, b, c, c
1218   __ ldr(r0, at_tos_p2());  // load b
1219   __ str(r2, at_tos_p2());  // store a in b
1220   // stack: ..., c, a, c, c
1221   __ str(r0, at_tos_p1());  // store b in c
1222   // stack: ..., c, a, b, c
1223 }
1224 
1225 void TemplateTable::dup2()
1226 {
1227   transition(vtos, vtos);
1228   // stack: ..., a, b
1229   __ ldr(r0, at_tos_p1());  // load a
1230   __ push(r0);                  // push a
1231   __ ldr(r0, at_tos_p1());  // load b
1232   __ push(r0);                  // push b
1233   // stack: ..., a, b, a, b
1234 }
1235 
1236 void TemplateTable::dup2_x1()
1237 {
1238   transition(vtos, vtos);
1239   // stack: ..., a, b, c
1240   __ ldr(r2, at_tos());  // load c
1241   __ ldr(r0, at_tos_p1());  // load b
1242   __ push(r0);                  // push b
1243   __ push(r2);                  // push c
1244   // stack: ..., a, b, c, b, c
1245   __ str(r2, at_tos_p3());  // store c in b
1246   // stack: ..., a, c, c, b, c
1247   __ ldr(r2, at_tos_p4());  // load a
1248   __ str(r2, at_tos_p2());  // store a in 2nd c
1249   // stack: ..., a, c, a, b, c
1250   __ str(r0, at_tos_p4());  // store b in a
1251   // stack: ..., b, c, a, b, c
1252 }
1253 
1254 void TemplateTable::dup2_x2()
1255 {
1256   transition(vtos, vtos);
1257   // stack: ..., a, b, c, d
1258   __ ldr(r2, at_tos());  // load d
1259   __ ldr(r0, at_tos_p1());  // load c
1260   __ push(r0)            ;      // push c
1261   __ push(r2);                  // push d
1262   // stack: ..., a, b, c, d, c, d
1263   __ ldr(r0, at_tos_p4());  // load b
1264   __ str(r0, at_tos_p2());  // store b in d
1265   __ str(r2, at_tos_p4());  // store d in b
1266   // stack: ..., a, d, c, b, c, d
1267   __ ldr(r2, at_tos_p5());  // load a
1268   __ ldr(r0, at_tos_p3());  // load c
1269   __ str(r2, at_tos_p3());  // store a in c
1270   __ str(r0, at_tos_p5());  // store c in a
1271   // stack: ..., c, d, a, b, c, d
1272 }
1273 
1274 void TemplateTable::swap()
1275 {
1276   transition(vtos, vtos);
1277   // stack: ..., a, b
1278   __ ldr(r2, at_tos_p1());  // load a
1279   __ ldr(r0, at_tos());  // load b
1280   __ str(r2, at_tos());  // store a in b
1281   __ str(r0, at_tos_p1());  // store b in a
1282   // stack: ..., b, a
1283 }
1284 
1285 void TemplateTable::iop2(Operation op)
1286 {
1287   transition(itos, itos);
1288   // r0 <== r1 op r0
1289   __ pop_i(r1);
1290   switch (op) {
1291   case add  : __ addw(r0, r1, r0); break;
1292   case sub  : __ subw(r0, r1, r0); break;
1293   case mul  : __ mulw(r0, r1, r0); break;
1294   case _and : __ andw(r0, r1, r0); break;
1295   case _or  : __ orrw(r0, r1, r0); break;
1296   case _xor : __ eorw(r0, r1, r0); break;
1297   case shl  : __ lslvw(r0, r1, r0); break;
1298   case shr  : __ asrvw(r0, r1, r0); break;
1299   case ushr : __ lsrvw(r0, r1, r0);break;
1300   default   : ShouldNotReachHere();
1301   }
1302 }
1303 
1304 void TemplateTable::lop2(Operation op)
1305 {
1306   transition(ltos, ltos);
1307   // r0 <== r1 op r0
1308   __ pop_l(r1);
1309   switch (op) {
1310   case add  : __ add(r0, r1, r0); break;
1311   case sub  : __ sub(r0, r1, r0); break;
1312   case mul  : __ mul(r0, r1, r0); break;
1313   case _and : __ andr(r0, r1, r0); break;
1314   case _or  : __ orr(r0, r1, r0); break;
1315   case _xor : __ eor(r0, r1, r0); break;
1316   default   : ShouldNotReachHere();
1317   }
1318 }
1319 
1320 void TemplateTable::idiv()
1321 {
1322   transition(itos, itos);
1323   // explicitly check for div0
1324   Label no_div0;
1325   __ cbnzw(r0, no_div0);
1326   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1327   __ br(rscratch1);
1328   __ bind(no_div0);
1329   __ pop_i(r1);
1330   // r0 <== r1 idiv r0
1331   __ corrected_idivl(r0, r1, r0, /* want_remainder */ false);
1332 }
1333 
1334 void TemplateTable::irem()
1335 {
1336   transition(itos, itos);
1337   // explicitly check for div0
1338   Label no_div0;
1339   __ cbnzw(r0, no_div0);
1340   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1341   __ br(rscratch1);
1342   __ bind(no_div0);
1343   __ pop_i(r1);
1344   // r0 <== r1 irem r0
1345   __ corrected_idivl(r0, r1, r0, /* want_remainder */ true);
1346 }
1347 
1348 void TemplateTable::lmul()
1349 {
1350   transition(ltos, ltos);
1351   __ pop_l(r1);
1352   __ mul(r0, r0, r1);
1353 }
1354 
1355 void TemplateTable::ldiv()
1356 {
1357   transition(ltos, ltos);
1358   // explicitly check for div0
1359   Label no_div0;
1360   __ cbnz(r0, no_div0);
1361   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1362   __ br(rscratch1);
1363   __ bind(no_div0);
1364   __ pop_l(r1);
1365   // r0 <== r1 ldiv r0
1366   __ corrected_idivq(r0, r1, r0, /* want_remainder */ false);
1367 }
1368 
1369 void TemplateTable::lrem()
1370 {
1371   transition(ltos, ltos);
1372   // explicitly check for div0
1373   Label no_div0;
1374   __ cbnz(r0, no_div0);
1375   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1376   __ br(rscratch1);
1377   __ bind(no_div0);
1378   __ pop_l(r1);
1379   // r0 <== r1 lrem r0
1380   __ corrected_idivq(r0, r1, r0, /* want_remainder */ true);
1381 }
1382 
1383 void TemplateTable::lshl()
1384 {
1385   transition(itos, ltos);
1386   // shift count is in r0
1387   __ pop_l(r1);
1388   __ lslv(r0, r1, r0);
1389 }
1390 
1391 void TemplateTable::lshr()
1392 {
1393   transition(itos, ltos);
1394   // shift count is in r0
1395   __ pop_l(r1);
1396   __ asrv(r0, r1, r0);
1397 }
1398 
1399 void TemplateTable::lushr()
1400 {
1401   transition(itos, ltos);
1402   // shift count is in r0
1403   __ pop_l(r1);
1404   __ lsrv(r0, r1, r0);
1405 }
1406 
1407 void TemplateTable::fop2(Operation op)
1408 {
1409   transition(ftos, ftos);
1410   switch (op) {
1411   case add:
1412     // n.b. use ldrd because this is a 64 bit slot
1413     __ pop_f(v1);
1414     __ fadds(v0, v1, v0);
1415     break;
1416   case sub:
1417     __ pop_f(v1);
1418     __ fsubs(v0, v1, v0);
1419     break;
1420   case mul:
1421     __ pop_f(v1);
1422     __ fmuls(v0, v1, v0);
1423     break;
1424   case div:
1425     __ pop_f(v1);
1426     __ fdivs(v0, v1, v0);
1427     break;
1428   case rem:
1429     __ fmovs(v1, v0);
1430     __ pop_f(v0);
1431     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem));
1432     break;
1433   default:
1434     ShouldNotReachHere();
1435     break;
1436   }
1437 }
1438 
1439 void TemplateTable::dop2(Operation op)
1440 {
1441   transition(dtos, dtos);
1442   switch (op) {
1443   case add:
1444     // n.b. use ldrd because this is a 64 bit slot
1445     __ pop_d(v1);
1446     __ faddd(v0, v1, v0);
1447     break;
1448   case sub:
1449     __ pop_d(v1);
1450     __ fsubd(v0, v1, v0);
1451     break;
1452   case mul:
1453     __ pop_d(v1);
1454     __ fmuld(v0, v1, v0);
1455     break;
1456   case div:
1457     __ pop_d(v1);
1458     __ fdivd(v0, v1, v0);
1459     break;
1460   case rem:
1461     __ fmovd(v1, v0);
1462     __ pop_d(v0);
1463     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem));
1464     break;
1465   default:
1466     ShouldNotReachHere();
1467     break;
1468   }
1469 }
1470 
1471 void TemplateTable::ineg()
1472 {
1473   transition(itos, itos);
1474   __ negw(r0, r0);
1475 
1476 }
1477 
1478 void TemplateTable::lneg()
1479 {
1480   transition(ltos, ltos);
1481   __ neg(r0, r0);
1482 }
1483 
1484 void TemplateTable::fneg()
1485 {
1486   transition(ftos, ftos);
1487   __ fnegs(v0, v0);
1488 }
1489 
1490 void TemplateTable::dneg()
1491 {
1492   transition(dtos, dtos);
1493   __ fnegd(v0, v0);
1494 }
1495 
1496 void TemplateTable::iinc()
1497 {
1498   transition(vtos, vtos);
1499   __ load_signed_byte(r1, at_bcp(2)); // get constant
1500   locals_index(r2);
1501   __ ldr(r0, iaddress(r2));
1502   __ addw(r0, r0, r1);
1503   __ str(r0, iaddress(r2));
1504 }
1505 
1506 void TemplateTable::wide_iinc()
1507 {
1508   transition(vtos, vtos);
1509   // __ mov(r1, zr);
1510   __ ldrw(r1, at_bcp(2)); // get constant and index
1511   __ rev16(r1, r1);
1512   __ ubfx(r2, r1, 0, 16);
1513   __ neg(r2, r2);
1514   __ sbfx(r1, r1, 16, 16);
1515   __ ldr(r0, iaddress(r2));
1516   __ addw(r0, r0, r1);
1517   __ str(r0, iaddress(r2));
1518 }
1519 
1520 void TemplateTable::convert()
1521 {
1522   // Checking
1523 #ifdef ASSERT
1524   {
1525     TosState tos_in  = ilgl;
1526     TosState tos_out = ilgl;
1527     switch (bytecode()) {
1528     case Bytecodes::_i2l: // fall through
1529     case Bytecodes::_i2f: // fall through
1530     case Bytecodes::_i2d: // fall through
1531     case Bytecodes::_i2b: // fall through
1532     case Bytecodes::_i2c: // fall through
1533     case Bytecodes::_i2s: tos_in = itos; break;
1534     case Bytecodes::_l2i: // fall through
1535     case Bytecodes::_l2f: // fall through
1536     case Bytecodes::_l2d: tos_in = ltos; break;
1537     case Bytecodes::_f2i: // fall through
1538     case Bytecodes::_f2l: // fall through
1539     case Bytecodes::_f2d: tos_in = ftos; break;
1540     case Bytecodes::_d2i: // fall through
1541     case Bytecodes::_d2l: // fall through
1542     case Bytecodes::_d2f: tos_in = dtos; break;
1543     default             : ShouldNotReachHere();
1544     }
1545     switch (bytecode()) {
1546     case Bytecodes::_l2i: // fall through
1547     case Bytecodes::_f2i: // fall through
1548     case Bytecodes::_d2i: // fall through
1549     case Bytecodes::_i2b: // fall through
1550     case Bytecodes::_i2c: // fall through
1551     case Bytecodes::_i2s: tos_out = itos; break;
1552     case Bytecodes::_i2l: // fall through
1553     case Bytecodes::_f2l: // fall through
1554     case Bytecodes::_d2l: tos_out = ltos; break;
1555     case Bytecodes::_i2f: // fall through
1556     case Bytecodes::_l2f: // fall through
1557     case Bytecodes::_d2f: tos_out = ftos; break;
1558     case Bytecodes::_i2d: // fall through
1559     case Bytecodes::_l2d: // fall through
1560     case Bytecodes::_f2d: tos_out = dtos; break;
1561     default             : ShouldNotReachHere();
1562     }
1563     transition(tos_in, tos_out);
1564   }
1565 #endif // ASSERT
1566   // static const int64_t is_nan = 0x8000000000000000L;
1567 
1568   // Conversion
1569   switch (bytecode()) {
1570   case Bytecodes::_i2l:
1571     __ sxtw(r0, r0);
1572     break;
1573   case Bytecodes::_i2f:
1574     __ scvtfws(v0, r0);
1575     break;
1576   case Bytecodes::_i2d:
1577     __ scvtfwd(v0, r0);
1578     break;
1579   case Bytecodes::_i2b:
1580     __ sxtbw(r0, r0);
1581     break;
1582   case Bytecodes::_i2c:
1583     __ uxthw(r0, r0);
1584     break;
1585   case Bytecodes::_i2s:
1586     __ sxthw(r0, r0);
1587     break;
1588   case Bytecodes::_l2i:
1589     __ uxtw(r0, r0);
1590     break;
1591   case Bytecodes::_l2f:
1592     __ scvtfs(v0, r0);
1593     break;
1594   case Bytecodes::_l2d:
1595     __ scvtfd(v0, r0);
1596     break;
1597   case Bytecodes::_f2i:
1598   {
1599     Label L_Okay;
1600     __ clear_fpsr();
1601     __ fcvtzsw(r0, v0);
1602     __ get_fpsr(r1);
1603     __ cbzw(r1, L_Okay);
1604     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2i));
1605     __ bind(L_Okay);
1606   }
1607     break;
1608   case Bytecodes::_f2l:
1609   {
1610     Label L_Okay;
1611     __ clear_fpsr();
1612     __ fcvtzs(r0, v0);
1613     __ get_fpsr(r1);
1614     __ cbzw(r1, L_Okay);
1615     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2l));
1616     __ bind(L_Okay);
1617   }
1618     break;
1619   case Bytecodes::_f2d:
1620     __ fcvts(v0, v0);
1621     break;
1622   case Bytecodes::_d2i:
1623   {
1624     Label L_Okay;
1625     __ clear_fpsr();
1626     __ fcvtzdw(r0, v0);
1627     __ get_fpsr(r1);
1628     __ cbzw(r1, L_Okay);
1629     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
1630     __ bind(L_Okay);
1631   }
1632     break;
1633   case Bytecodes::_d2l:
1634   {
1635     Label L_Okay;
1636     __ clear_fpsr();
1637     __ fcvtzd(r0, v0);
1638     __ get_fpsr(r1);
1639     __ cbzw(r1, L_Okay);
1640     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
1641     __ bind(L_Okay);
1642   }
1643     break;
1644   case Bytecodes::_d2f:
1645     __ fcvtd(v0, v0);
1646     break;
1647   default:
1648     ShouldNotReachHere();
1649   }
1650 }
1651 
1652 void TemplateTable::lcmp()
1653 {
1654   transition(ltos, itos);
1655   Label done;
1656   __ pop_l(r1);
1657   __ cmp(r1, r0);
1658   __ mov(r0, (u_int64_t)-1L);
1659   __ br(Assembler::LT, done);
1660   // __ mov(r0, 1UL);
1661   // __ csel(r0, r0, zr, Assembler::NE);
1662   // and here is a faster way
1663   __ csinc(r0, zr, zr, Assembler::EQ);
1664   __ bind(done);
1665 }
1666 
1667 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1668 {
1669   Label done;
1670   if (is_float) {
1671     // XXX get rid of pop here, use ... reg, mem32
1672     __ pop_f(v1);
1673     __ fcmps(v1, v0);
1674   } else {
1675     // XXX get rid of pop here, use ... reg, mem64
1676     __ pop_d(v1);
1677     __ fcmpd(v1, v0);
1678   }
1679   if (unordered_result < 0) {
1680     // we want -1 for unordered or less than, 0 for equal and 1 for
1681     // greater than.
1682     __ mov(r0, (u_int64_t)-1L);
1683     // for FP LT tests less than or unordered
1684     __ br(Assembler::LT, done);
1685     // install 0 for EQ otherwise 1
1686     __ csinc(r0, zr, zr, Assembler::EQ);
1687   } else {
1688     // we want -1 for less than, 0 for equal and 1 for unordered or
1689     // greater than.
1690     __ mov(r0, 1L);
1691     // for FP HI tests greater than or unordered
1692     __ br(Assembler::HI, done);
1693     // install 0 for EQ otherwise ~0
1694     __ csinv(r0, zr, zr, Assembler::EQ);
1695 
1696   }
1697   __ bind(done);
1698 }
1699 
1700 void TemplateTable::branch(bool is_jsr, bool is_wide)
1701 {
1702   // We might be moving to a safepoint.  The thread which calls
1703   // Interpreter::notice_safepoints() will effectively flush its cache
1704   // when it makes a system call, but we need to do something to
1705   // ensure that we see the changed dispatch table.
1706   __ membar(MacroAssembler::LoadLoad);
1707 
1708   __ profile_taken_branch(r0, r1);
1709   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1710                              InvocationCounter::counter_offset();
1711   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1712                               InvocationCounter::counter_offset();
1713 
1714   // load branch displacement
1715   if (!is_wide) {
1716     __ ldrh(r2, at_bcp(1));
1717     __ rev16(r2, r2);
1718     // sign extend the 16 bit value in r2
1719     __ sbfm(r2, r2, 0, 15);
1720   } else {
1721     __ ldrw(r2, at_bcp(1));
1722     __ revw(r2, r2);
1723     // sign extend the 32 bit value in r2
1724     __ sbfm(r2, r2, 0, 31);
1725   }
1726 
1727   // Handle all the JSR stuff here, then exit.
1728   // It's much shorter and cleaner than intermingling with the non-JSR
1729   // normal-branch stuff occurring below.
1730 
1731   if (is_jsr) {
1732     // Pre-load the next target bytecode into rscratch1
1733     __ load_unsigned_byte(rscratch1, Address(rbcp, r2));
1734     // compute return address as bci
1735     __ ldr(rscratch2, Address(rmethod, Method::const_offset()));
1736     __ add(rscratch2, rscratch2,
1737            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1738     __ sub(r1, rbcp, rscratch2);
1739     __ push_i(r1);
1740     // Adjust the bcp by the 16-bit displacement in r2
1741     __ add(rbcp, rbcp, r2);
1742     __ dispatch_only(vtos);
1743     return;
1744   }
1745 
1746   // Normal (non-jsr) branch handling
1747 
1748   // Adjust the bcp by the displacement in r2
1749   __ add(rbcp, rbcp, r2);
1750 
1751   assert(UseLoopCounter || !UseOnStackReplacement,
1752          "on-stack-replacement requires loop counters");
1753   Label backedge_counter_overflow;
1754   Label profile_method;
1755   Label dispatch;
1756   if (UseLoopCounter) {
1757     // increment backedge counter for backward branches
1758     // r0: MDO
1759     // w1: MDO bumped taken-count
1760     // r2: target offset
1761     __ cmp(r2, zr);
1762     __ br(Assembler::GT, dispatch); // count only if backward branch
1763 
1764     // ECN: FIXME: This code smells
1765     // check if MethodCounters exists
1766     Label has_counters;
1767     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1768     __ cbnz(rscratch1, has_counters);
1769     __ push(r0);
1770     __ push(r1);
1771     __ push(r2);
1772     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1773             InterpreterRuntime::build_method_counters), rmethod);
1774     __ pop(r2);
1775     __ pop(r1);
1776     __ pop(r0);
1777     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1778     __ cbz(rscratch1, dispatch); // No MethodCounters allocated, OutOfMemory
1779     __ bind(has_counters);
1780 
1781     if (TieredCompilation) {
1782       Label no_mdo;
1783       int increment = InvocationCounter::count_increment;
1784       int mask = ((1 << Tier0BackedgeNotifyFreqLog) - 1) << InvocationCounter::count_shift;
1785       if (ProfileInterpreter) {
1786         // Are we profiling?
1787         __ ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
1788         __ cbz(r1, no_mdo);
1789         // Increment the MDO backedge counter
1790         const Address mdo_backedge_counter(r1, in_bytes(MethodData::backedge_counter_offset()) +
1791                                            in_bytes(InvocationCounter::counter_offset()));
1792         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1793                                    r0, rscratch2, false, Assembler::EQ, &backedge_counter_overflow);
1794         __ b(dispatch);
1795       }
1796       __ bind(no_mdo);
1797       // Increment backedge counter in MethodCounters*
1798       __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1799       __ increment_mask_and_jump(Address(rscratch1, be_offset), increment, mask,
1800                                  r0, rscratch2, false, Assembler::EQ, &backedge_counter_overflow);
1801     } else {
1802       // increment counter
1803       __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
1804       __ ldrw(r0, Address(rscratch2, be_offset));        // load backedge counter
1805       __ addw(rscratch1, r0, InvocationCounter::count_increment); // increment counter
1806       __ strw(rscratch1, Address(rscratch2, be_offset));        // store counter
1807 
1808       __ ldrw(r0, Address(rscratch2, inv_offset));    // load invocation counter
1809       __ andw(r0, r0, (unsigned)InvocationCounter::count_mask_value); // and the status bits
1810       __ addw(r0, r0, rscratch1);        // add both counters
1811 
1812       if (ProfileInterpreter) {
1813         // Test to see if we should create a method data oop
1814         __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterProfileLimit));
1815         __ ldrw(rscratch1, rscratch1);
1816         __ cmpw(r0, rscratch1);
1817         __ br(Assembler::LT, dispatch);
1818 
1819         // if no method data exists, go to profile method
1820         __ test_method_data_pointer(r0, profile_method);
1821 
1822         if (UseOnStackReplacement) {
1823           // check for overflow against w1 which is the MDO taken count
1824           __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
1825           __ ldrw(rscratch1, rscratch1);
1826           __ cmpw(r1, rscratch1);
1827           __ br(Assembler::LO, dispatch); // Intel == Assembler::below
1828 
1829           // When ProfileInterpreter is on, the backedge_count comes
1830           // from the MethodData*, which value does not get reset on
1831           // the call to frequency_counter_overflow().  To avoid
1832           // excessive calls to the overflow routine while the method is
1833           // being compiled, add a second test to make sure the overflow
1834           // function is called only once every overflow_frequency.
1835           const int overflow_frequency = 1024;
1836           __ andsw(r1, r1, overflow_frequency - 1);
1837           __ br(Assembler::EQ, backedge_counter_overflow);
1838 
1839         }
1840       } else {
1841         if (UseOnStackReplacement) {
1842           // check for overflow against w0, which is the sum of the
1843           // counters
1844           __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
1845           __ ldrw(rscratch1, rscratch1);
1846           __ cmpw(r0, rscratch1);
1847           __ br(Assembler::HS, backedge_counter_overflow); // Intel == Assembler::aboveEqual
1848         }
1849       }
1850     }
1851   }
1852   __ bind(dispatch);
1853 
1854   // Pre-load the next target bytecode into rscratch1
1855   __ load_unsigned_byte(rscratch1, Address(rbcp, 0));
1856 
1857   // continue with the bytecode @ target
1858   // rscratch1: target bytecode
1859   // rbcp: target bcp
1860   __ dispatch_only(vtos);
1861 
1862   if (UseLoopCounter) {
1863     if (ProfileInterpreter) {
1864       // Out-of-line code to allocate method data oop.
1865       __ bind(profile_method);
1866       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1867       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1868       __ set_method_data_pointer_for_bcp();
1869       __ b(dispatch);
1870     }
1871 
1872     if (TieredCompilation || UseOnStackReplacement) {
1873       // invocation counter overflow
1874       __ bind(backedge_counter_overflow);
1875       __ neg(r2, r2);
1876       __ add(r2, r2, rbcp);     // branch bcp
1877       // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
1878       __ call_VM(noreg,
1879                  CAST_FROM_FN_PTR(address,
1880                                   InterpreterRuntime::frequency_counter_overflow),
1881                  r2);
1882       if (!UseOnStackReplacement)
1883         __ b(dispatch);
1884     }
1885 
1886     if (UseOnStackReplacement) {
1887       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1888 
1889       // r0: osr nmethod (osr ok) or NULL (osr not possible)
1890       // w1: target bytecode
1891       // r2: scratch
1892       __ cbz(r0, dispatch);     // test result -- no osr if null
1893       // nmethod may have been invalidated (VM may block upon call_VM return)
1894       __ ldrw(r2, Address(r0, nmethod::entry_bci_offset()));
1895       // InvalidOSREntryBci == -2 which overflows cmpw as unsigned
1896       // use cmnw against -InvalidOSREntryBci which does the same thing
1897       __ cmn(r2, -InvalidOSREntryBci);
1898       __ br(Assembler::EQ, dispatch);
1899 
1900       // We have the address of an on stack replacement routine in r0
1901       // We need to prepare to execute the OSR method. First we must
1902       // migrate the locals and monitors off of the stack.
1903 
1904       __ mov(r19, r0);                             // save the nmethod
1905 
1906       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
1907 
1908       // r0 is OSR buffer, move it to expected parameter location
1909       __ mov(j_rarg0, r0);
1910 
1911       // remove activation
1912       // get sender esp
1913       __ ldr(esp,
1914           Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
1915       // remove frame anchor
1916       __ leave();
1917       // Ensure compiled code always sees stack at proper alignment
1918       __ andr(sp, esp, -16);
1919 
1920       // and begin the OSR nmethod
1921       __ ldr(rscratch1, Address(r19, nmethod::osr_entry_point_offset()));
1922       __ br(rscratch1);
1923     }
1924   }
1925 }
1926 
1927 
1928 void TemplateTable::if_0cmp(Condition cc)
1929 {
1930   transition(itos, vtos);
1931   // assume branch is more often taken than not (loops use backward branches)
1932   Label not_taken;
1933   if (cc == equal)
1934     __ cbnzw(r0, not_taken);
1935   else if (cc == not_equal)
1936     __ cbzw(r0, not_taken);
1937   else {
1938     __ andsw(zr, r0, r0);
1939     __ br(j_not(cc), not_taken);
1940   }
1941 
1942   branch(false, false);
1943   __ bind(not_taken);
1944   __ profile_not_taken_branch(r0);
1945 }
1946 
1947 void TemplateTable::if_icmp(Condition cc)
1948 {
1949   transition(itos, vtos);
1950   // assume branch is more often taken than not (loops use backward branches)
1951   Label not_taken;
1952   __ pop_i(r1);
1953   __ cmpw(r1, r0, Assembler::LSL);
1954   __ br(j_not(cc), not_taken);
1955   branch(false, false);
1956   __ bind(not_taken);
1957   __ profile_not_taken_branch(r0);
1958 }
1959 
1960 void TemplateTable::if_nullcmp(Condition cc)
1961 {
1962   transition(atos, vtos);
1963   // assume branch is more often taken than not (loops use backward branches)
1964   Label not_taken;
1965   if (cc == equal)
1966     __ cbnz(r0, not_taken);
1967   else
1968     __ cbz(r0, not_taken);
1969   branch(false, false);
1970   __ bind(not_taken);
1971   __ profile_not_taken_branch(r0);
1972 }
1973 
1974 void TemplateTable::if_acmp(Condition cc)
1975 {
1976   transition(atos, vtos);
1977   // assume branch is more often taken than not (loops use backward branches)
1978   Label not_taken;
1979   __ pop_ptr(r1);
1980   __ cmp(r1, r0);
1981   __ br(j_not(cc), not_taken);
1982   branch(false, false);
1983   __ bind(not_taken);
1984   __ profile_not_taken_branch(r0);
1985 }
1986 
1987 void TemplateTable::ret() {
1988   transition(vtos, vtos);
1989   // We might be moving to a safepoint.  The thread which calls
1990   // Interpreter::notice_safepoints() will effectively flush its cache
1991   // when it makes a system call, but we need to do something to
1992   // ensure that we see the changed dispatch table.
1993   __ membar(MacroAssembler::LoadLoad);
1994 
1995   locals_index(r1);
1996   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
1997   __ profile_ret(r1, r2);
1998   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
1999   __ lea(rbcp, Address(rbcp, r1));
2000   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2001   __ dispatch_next(vtos);
2002 }
2003 
2004 void TemplateTable::wide_ret() {
2005   transition(vtos, vtos);
2006   locals_index_wide(r1);
2007   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2008   __ profile_ret(r1, r2);
2009   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2010   __ lea(rbcp, Address(rbcp, r1));
2011   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2012   __ dispatch_next(vtos);
2013 }
2014 
2015 
2016 void TemplateTable::tableswitch() {
2017   Label default_case, continue_execution;
2018   transition(itos, vtos);
2019   // align rbcp
2020   __ lea(r1, at_bcp(BytesPerInt));
2021   __ andr(r1, r1, -BytesPerInt);
2022   // load lo & hi
2023   __ ldrw(r2, Address(r1, BytesPerInt));
2024   __ ldrw(r3, Address(r1, 2 * BytesPerInt));
2025   __ rev32(r2, r2);
2026   __ rev32(r3, r3);
2027   // check against lo & hi
2028   __ cmpw(r0, r2);
2029   __ br(Assembler::LT, default_case);
2030   __ cmpw(r0, r3);
2031   __ br(Assembler::GT, default_case);
2032   // lookup dispatch offset
2033   __ subw(r0, r0, r2);
2034   __ lea(r3, Address(r1, r0, Address::uxtw(2)));
2035   __ ldrw(r3, Address(r3, 3 * BytesPerInt));
2036   __ profile_switch_case(r0, r1, r2);
2037   // continue execution
2038   __ bind(continue_execution);
2039   __ rev32(r3, r3);
2040   __ load_unsigned_byte(rscratch1, Address(rbcp, r3, Address::sxtw(0)));
2041   __ add(rbcp, rbcp, r3, ext::sxtw);
2042   __ dispatch_only(vtos);
2043   // handle default
2044   __ bind(default_case);
2045   __ profile_switch_default(r0);
2046   __ ldrw(r3, Address(r1, 0));
2047   __ b(continue_execution);
2048 }
2049 
2050 void TemplateTable::lookupswitch() {
2051   transition(itos, itos);
2052   __ stop("lookupswitch bytecode should have been rewritten");
2053 }
2054 
2055 void TemplateTable::fast_linearswitch() {
2056   transition(itos, vtos);
2057   Label loop_entry, loop, found, continue_execution;
2058   // bswap r0 so we can avoid bswapping the table entries
2059   __ rev32(r0, r0);
2060   // align rbcp
2061   __ lea(r19, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2062                                     // this instruction (change offsets
2063                                     // below)
2064   __ andr(r19, r19, -BytesPerInt);
2065   // set counter
2066   __ ldrw(r1, Address(r19, BytesPerInt));
2067   __ rev32(r1, r1);
2068   __ b(loop_entry);
2069   // table search
2070   __ bind(loop);
2071   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2072   __ ldrw(rscratch1, Address(rscratch1, 2 * BytesPerInt));
2073   __ cmpw(r0, rscratch1);
2074   __ br(Assembler::EQ, found);
2075   __ bind(loop_entry);
2076   __ subs(r1, r1, 1);
2077   __ br(Assembler::PL, loop);
2078   // default case
2079   __ profile_switch_default(r0);
2080   __ ldrw(r3, Address(r19, 0));
2081   __ b(continue_execution);
2082   // entry found -> get offset
2083   __ bind(found);
2084   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2085   __ ldrw(r3, Address(rscratch1, 3 * BytesPerInt));
2086   __ profile_switch_case(r1, r0, r19);
2087   // continue execution
2088   __ bind(continue_execution);
2089   __ rev32(r3, r3);
2090   __ add(rbcp, rbcp, r3, ext::sxtw);
2091   __ ldrb(rscratch1, Address(rbcp, 0));
2092   __ dispatch_only(vtos);
2093 }
2094 
2095 void TemplateTable::fast_binaryswitch() {
2096   transition(itos, vtos);
2097   // Implementation using the following core algorithm:
2098   //
2099   // int binary_search(int key, LookupswitchPair* array, int n) {
2100   //   // Binary search according to "Methodik des Programmierens" by
2101   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2102   //   int i = 0;
2103   //   int j = n;
2104   //   while (i+1 < j) {
2105   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2106   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2107   //     // where a stands for the array and assuming that the (inexisting)
2108   //     // element a[n] is infinitely big.
2109   //     int h = (i + j) >> 1;
2110   //     // i < h < j
2111   //     if (key < array[h].fast_match()) {
2112   //       j = h;
2113   //     } else {
2114   //       i = h;
2115   //     }
2116   //   }
2117   //   // R: a[i] <= key < a[i+1] or Q
2118   //   // (i.e., if key is within array, i is the correct index)
2119   //   return i;
2120   // }
2121 
2122   // Register allocation
2123   const Register key   = r0; // already set (tosca)
2124   const Register array = r1;
2125   const Register i     = r2;
2126   const Register j     = r3;
2127   const Register h     = rscratch1;
2128   const Register temp  = rscratch2;
2129 
2130   // Find array start
2131   __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2132                                           // get rid of this
2133                                           // instruction (change
2134                                           // offsets below)
2135   __ andr(array, array, -BytesPerInt);
2136 
2137   // Initialize i & j
2138   __ mov(i, 0);                            // i = 0;
2139   __ ldrw(j, Address(array, -BytesPerInt)); // j = length(array);
2140 
2141   // Convert j into native byteordering
2142   __ rev32(j, j);
2143 
2144   // And start
2145   Label entry;
2146   __ b(entry);
2147 
2148   // binary search loop
2149   {
2150     Label loop;
2151     __ bind(loop);
2152     // int h = (i + j) >> 1;
2153     __ addw(h, i, j);                           // h = i + j;
2154     __ lsrw(h, h, 1);                                   // h = (i + j) >> 1;
2155     // if (key < array[h].fast_match()) {
2156     //   j = h;
2157     // } else {
2158     //   i = h;
2159     // }
2160     // Convert array[h].match to native byte-ordering before compare
2161     __ ldr(temp, Address(array, h, Address::lsl(3)));
2162     __ rev32(temp, temp);
2163     __ cmpw(key, temp);
2164     // j = h if (key <  array[h].fast_match())
2165     __ csel(j, h, j, Assembler::LT);
2166     // i = h if (key >= array[h].fast_match())
2167     __ csel(i, h, i, Assembler::GE);
2168     // while (i+1 < j)
2169     __ bind(entry);
2170     __ addw(h, i, 1);          // i+1
2171     __ cmpw(h, j);             // i+1 < j
2172     __ br(Assembler::LT, loop);
2173   }
2174 
2175   // end of binary search, result index is i (must check again!)
2176   Label default_case;
2177   // Convert array[i].match to native byte-ordering before compare
2178   __ ldr(temp, Address(array, i, Address::lsl(3)));
2179   __ rev32(temp, temp);
2180   __ cmpw(key, temp);
2181   __ br(Assembler::NE, default_case);
2182 
2183   // entry found -> j = offset
2184   __ add(j, array, i, ext::uxtx, 3);
2185   __ ldrw(j, Address(j, BytesPerInt));
2186   __ profile_switch_case(i, key, array);
2187   __ rev32(j, j);
2188   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2189   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2190   __ dispatch_only(vtos);
2191 
2192   // default case -> j = default offset
2193   __ bind(default_case);
2194   __ profile_switch_default(i);
2195   __ ldrw(j, Address(array, -2 * BytesPerInt));
2196   __ rev32(j, j);
2197   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2198   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2199   __ dispatch_only(vtos);
2200 }
2201 
2202 
2203 void TemplateTable::_return(TosState state)
2204 {
2205   transition(state, state);
2206   assert(_desc->calls_vm(),
2207          "inconsistent calls_vm information"); // call in remove_activation
2208 
2209   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2210     assert(state == vtos, "only valid state");
2211 
2212     __ ldr(c_rarg1, aaddress(0));
2213     __ load_klass(r3, c_rarg1);
2214     __ ldrw(r3, Address(r3, Klass::access_flags_offset()));
2215     __ tst(r3, JVM_ACC_HAS_FINALIZER);
2216     Label skip_register_finalizer;
2217     __ br(Assembler::EQ, skip_register_finalizer);
2218 
2219     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2220 
2221     __ bind(skip_register_finalizer);
2222   }
2223 
2224   // Issue a StoreStore barrier after all stores but before return
2225   // from any constructor for any class with a final field.  We don't
2226   // know if this is a finalizer, so we always do so.
2227   if (_desc->bytecode() == Bytecodes::_return)
2228     __ membar(MacroAssembler::StoreStore);
2229 
2230   // Narrow result if state is itos but result type is smaller.
2231   // Need to narrow in the return bytecode rather than in generate_return_entry
2232   // since compiled code callers expect the result to already be narrowed.
2233   if (state == itos) {
2234     __ narrow(r0);
2235   }
2236 
2237   __ remove_activation(state);
2238   __ ret(lr);
2239 }
2240 
2241 // ----------------------------------------------------------------------------
2242 // Volatile variables demand their effects be made known to all CPU's
2243 // in order.  Store buffers on most chips allow reads & writes to
2244 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2245 // without some kind of memory barrier (i.e., it's not sufficient that
2246 // the interpreter does not reorder volatile references, the hardware
2247 // also must not reorder them).
2248 //
2249 // According to the new Java Memory Model (JMM):
2250 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2251 //     writes act as aquire & release, so:
2252 // (2) A read cannot let unrelated NON-volatile memory refs that
2253 //     happen after the read float up to before the read.  It's OK for
2254 //     non-volatile memory refs that happen before the volatile read to
2255 //     float down below it.
2256 // (3) Similar a volatile write cannot let unrelated NON-volatile
2257 //     memory refs that happen BEFORE the write float down to after the
2258 //     write.  It's OK for non-volatile memory refs that happen after the
2259 //     volatile write to float up before it.
2260 //
2261 // We only put in barriers around volatile refs (they are expensive),
2262 // not _between_ memory refs (that would require us to track the
2263 // flavor of the previous memory refs).  Requirements (2) and (3)
2264 // require some barriers before volatile stores and after volatile
2265 // loads.  These nearly cover requirement (1) but miss the
2266 // volatile-store-volatile-load case.  This final case is placed after
2267 // volatile-stores although it could just as well go before
2268 // volatile-loads.
2269 
2270 void TemplateTable::resolve_cache_and_index(int byte_no,
2271                                             Register Rcache,
2272                                             Register index,
2273                                             size_t index_size) {
2274   const Register temp = r19;
2275   assert_different_registers(Rcache, index, temp);
2276 
2277   Label resolved;
2278   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2279   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2280   __ cmp(temp, (int) bytecode());  // have we resolved this bytecode?
2281   __ br(Assembler::EQ, resolved);
2282 
2283   // resolve first time through
2284   address entry;
2285   switch (bytecode()) {
2286   case Bytecodes::_getstatic:
2287   case Bytecodes::_putstatic:
2288   case Bytecodes::_getfield:
2289   case Bytecodes::_putfield:
2290     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_get_put);
2291     break;
2292   case Bytecodes::_invokevirtual:
2293   case Bytecodes::_invokespecial:
2294   case Bytecodes::_invokestatic:
2295   case Bytecodes::_invokeinterface:
2296     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invoke);
2297     break;
2298   case Bytecodes::_invokehandle:
2299     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invokehandle);
2300     break;
2301   case Bytecodes::_invokedynamic:
2302     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invokedynamic);
2303     break;
2304   default:
2305     fatal(err_msg("unexpected bytecode: %s", Bytecodes::name(bytecode())));
2306     break;
2307   }
2308   __ mov(temp, (int) bytecode());
2309   __ call_VM(noreg, entry, temp);
2310 
2311   // Update registers with resolved info
2312   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2313   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2314   // so all clients ofthis method must be modified accordingly
2315   __ bind(resolved);
2316 }
2317 
2318 // The Rcache and index registers must be set before call
2319 // n.b unlike x86 cache already includes the index offset
2320 void TemplateTable::load_field_cp_cache_entry(Register obj,
2321                                               Register cache,
2322                                               Register index,
2323                                               Register off,
2324                                               Register flags,
2325                                               bool is_static = false) {
2326   assert_different_registers(cache, index, flags, off);
2327 
2328   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2329   // Field offset
2330   __ ldr(off, Address(cache, in_bytes(cp_base_offset +
2331                                           ConstantPoolCacheEntry::f2_offset())));
2332   // Flags
2333   __ ldrw(flags, Address(cache, in_bytes(cp_base_offset +
2334                                            ConstantPoolCacheEntry::flags_offset())));
2335 
2336   // klass overwrite register
2337   if (is_static) {
2338     __ ldr(obj, Address(cache, in_bytes(cp_base_offset +
2339                                         ConstantPoolCacheEntry::f1_offset())));
2340     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2341     __ ldr(obj, Address(obj, mirror_offset));
2342   }
2343 }
2344 
2345 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2346                                                Register method,
2347                                                Register itable_index,
2348                                                Register flags,
2349                                                bool is_invokevirtual,
2350                                                bool is_invokevfinal, /*unused*/
2351                                                bool is_invokedynamic) {
2352   // setup registers
2353   const Register cache = rscratch2;
2354   const Register index = r4;
2355   assert_different_registers(method, flags);
2356   assert_different_registers(method, cache, index);
2357   assert_different_registers(itable_index, flags);
2358   assert_different_registers(itable_index, cache, index);
2359   // determine constant pool cache field offsets
2360   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2361   const int method_offset = in_bytes(
2362     ConstantPoolCache::base_offset() +
2363       (is_invokevirtual
2364        ? ConstantPoolCacheEntry::f2_offset()
2365        : ConstantPoolCacheEntry::f1_offset()));
2366   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2367                                     ConstantPoolCacheEntry::flags_offset());
2368   // access constant pool cache fields
2369   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2370                                     ConstantPoolCacheEntry::f2_offset());
2371 
2372   size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2373   resolve_cache_and_index(byte_no, cache, index, index_size);
2374   __ ldr(method, Address(cache, method_offset));
2375 
2376   if (itable_index != noreg) {
2377     __ ldr(itable_index, Address(cache, index_offset));
2378   }
2379   __ ldrw(flags, Address(cache, flags_offset));
2380 }
2381 
2382 
2383 // The registers cache and index expected to be set before call.
2384 // Correct values of the cache and index registers are preserved.
2385 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2386                                             bool is_static, bool has_tos) {
2387   // do the JVMTI work here to avoid disturbing the register state below
2388   // We use c_rarg registers here because we want to use the register used in
2389   // the call to the VM
2390   if (JvmtiExport::can_post_field_access()) {
2391     // Check to see if a field access watch has been set before we
2392     // take the time to call into the VM.
2393     Label L1;
2394     assert_different_registers(cache, index, r0);
2395     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2396     __ ldrw(r0, Address(rscratch1));
2397     __ cbzw(r0, L1);
2398 
2399     __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2400     __ lea(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2401 
2402     if (is_static) {
2403       __ mov(c_rarg1, zr); // NULL object reference
2404     } else {
2405       __ ldr(c_rarg1, at_tos()); // get object pointer without popping it
2406       __ verify_oop(c_rarg1);
2407     }
2408     // c_rarg1: object pointer or NULL
2409     // c_rarg2: cache entry pointer
2410     // c_rarg3: jvalue object on the stack
2411     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2412                                        InterpreterRuntime::post_field_access),
2413                c_rarg1, c_rarg2, c_rarg3);
2414     __ get_cache_and_index_at_bcp(cache, index, 1);
2415     __ bind(L1);
2416   }
2417 }
2418 
2419 void TemplateTable::pop_and_check_object(Register r)
2420 {
2421   __ pop_ptr(r);
2422   __ null_check(r);  // for field access must check obj.
2423   __ verify_oop(r);
2424 }
2425 
2426 void TemplateTable::getfield_or_static(int byte_no, bool is_static)
2427 {
2428   const Register cache = r2;
2429   const Register index = r3;
2430   const Register obj   = r4;
2431   const Register off   = r19;
2432   const Register flags = r0;
2433   const Register raw_flags = r6;
2434   const Register bc    = r4; // uses same reg as obj, so don't mix them
2435 
2436   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2437   jvmti_post_field_access(cache, index, is_static, false);
2438   load_field_cp_cache_entry(obj, cache, index, off, raw_flags, is_static);
2439 
2440   if (!is_static) {
2441     // obj is on the stack
2442     pop_and_check_object(obj);
2443   }
2444 
2445   // 8179954: We need to make sure that the code generated for
2446   // volatile accesses forms a sequentially-consistent set of
2447   // operations when combined with STLR and LDAR.  Without a leading
2448   // membar it's possible for a simple Dekker test to fail if loads
2449   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
2450   // the stores in one method and we interpret the loads in another.
2451   if (! UseBarriersForVolatile) {
2452     Label notVolatile;
2453     __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2454     __ membar(MacroAssembler::AnyAny);
2455     __ bind(notVolatile);
2456   }
2457 
2458   const Address field(obj, off);
2459 
2460   Label Done, notByte, notBool, notInt, notShort, notChar,
2461               notLong, notFloat, notObj, notDouble;
2462 
2463   // x86 uses a shift and mask or wings it with a shift plus assert
2464   // the mask is not needed. aarch64 just uses bitfield extract
2465   __ ubfxw(flags, raw_flags, ConstantPoolCacheEntry::tos_state_shift,
2466            ConstantPoolCacheEntry::tos_state_bits);
2467 
2468   assert(btos == 0, "change code, btos != 0");
2469   __ cbnz(flags, notByte);
2470 
2471   // btos
2472   __ load_signed_byte(r0, field);
2473   __ push(btos);
2474   // Rewrite bytecode to be faster
2475   if (!is_static) {
2476     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2477   }
2478   __ b(Done);
2479 
2480   __ bind(notByte);
2481   __ cmp(flags, ztos);
2482   __ br(Assembler::NE, notBool);
2483 
2484   // ztos (same code as btos)
2485   __ ldrsb(r0, field);
2486   __ push(ztos);
2487   // Rewrite bytecode to be faster
2488   if (!is_static) {
2489     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2490     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2491   }
2492   __ b(Done);
2493 
2494   __ bind(notBool);
2495   __ cmp(flags, atos);
2496   __ br(Assembler::NE, notObj);
2497   // atos
2498   __ load_heap_oop(r0, field);
2499   __ push(atos);
2500   if (!is_static) {
2501     patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2502   }
2503   __ b(Done);
2504 
2505   __ bind(notObj);
2506   __ cmp(flags, itos);
2507   __ br(Assembler::NE, notInt);
2508   // itos
2509   __ ldrw(r0, field);
2510   __ push(itos);
2511   // Rewrite bytecode to be faster
2512   if (!is_static) {
2513     patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2514   }
2515   __ b(Done);
2516 
2517   __ bind(notInt);
2518   __ cmp(flags, ctos);
2519   __ br(Assembler::NE, notChar);
2520   // ctos
2521   __ load_unsigned_short(r0, field);
2522   __ push(ctos);
2523   // Rewrite bytecode to be faster
2524   if (!is_static) {
2525     patch_bytecode(Bytecodes::_fast_cgetfield, bc, r1);
2526   }
2527   __ b(Done);
2528 
2529   __ bind(notChar);
2530   __ cmp(flags, stos);
2531   __ br(Assembler::NE, notShort);
2532   // stos
2533   __ load_signed_short(r0, field);
2534   __ push(stos);
2535   // Rewrite bytecode to be faster
2536   if (!is_static) {
2537     patch_bytecode(Bytecodes::_fast_sgetfield, bc, r1);
2538   }
2539   __ b(Done);
2540 
2541   __ bind(notShort);
2542   __ cmp(flags, ltos);
2543   __ br(Assembler::NE, notLong);
2544   // ltos
2545   __ ldr(r0, field);
2546   __ push(ltos);
2547   // Rewrite bytecode to be faster
2548   if (!is_static) {
2549     patch_bytecode(Bytecodes::_fast_lgetfield, bc, r1);
2550   }
2551   __ b(Done);
2552 
2553   __ bind(notLong);
2554   __ cmp(flags, ftos);
2555   __ br(Assembler::NE, notFloat);
2556   // ftos
2557   __ ldrs(v0, field);
2558   __ push(ftos);
2559   // Rewrite bytecode to be faster
2560   if (!is_static) {
2561     patch_bytecode(Bytecodes::_fast_fgetfield, bc, r1);
2562   }
2563   __ b(Done);
2564 
2565   __ bind(notFloat);
2566 #ifdef ASSERT
2567   __ cmp(flags, dtos);
2568   __ br(Assembler::NE, notDouble);
2569 #endif
2570   // dtos
2571   __ ldrd(v0, field);
2572   __ push(dtos);
2573   // Rewrite bytecode to be faster
2574   if (!is_static) {
2575     patch_bytecode(Bytecodes::_fast_dgetfield, bc, r1);
2576   }
2577 #ifdef ASSERT
2578   __ b(Done);
2579 
2580   __ bind(notDouble);
2581   __ stop("Bad state");
2582 #endif
2583 
2584   __ bind(Done);
2585 
2586   Label notVolatile;
2587   __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2588   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2589   __ bind(notVolatile);
2590 }
2591 
2592 
2593 void TemplateTable::getfield(int byte_no)
2594 {
2595   getfield_or_static(byte_no, false);
2596 }
2597 
2598 void TemplateTable::getstatic(int byte_no)
2599 {
2600   getfield_or_static(byte_no, true);
2601 }
2602 
2603 // The registers cache and index expected to be set before call.
2604 // The function may destroy various registers, just not the cache and index registers.
2605 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2606   transition(vtos, vtos);
2607 
2608   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2609 
2610   if (JvmtiExport::can_post_field_modification()) {
2611     // Check to see if a field modification watch has been set before
2612     // we take the time to call into the VM.
2613     Label L1;
2614     assert_different_registers(cache, index, r0);
2615     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2616     __ ldrw(r0, Address(rscratch1));
2617     __ cbz(r0, L1);
2618 
2619     __ get_cache_and_index_at_bcp(c_rarg2, rscratch1, 1);
2620 
2621     if (is_static) {
2622       // Life is simple.  Null out the object pointer.
2623       __ mov(c_rarg1, zr);
2624     } else {
2625       // Life is harder. The stack holds the value on top, followed by
2626       // the object.  We don't know the size of the value, though; it
2627       // could be one or two words depending on its type. As a result,
2628       // we must find the type to determine where the object is.
2629       __ ldrw(c_rarg3, Address(c_rarg2,
2630                                in_bytes(cp_base_offset +
2631                                         ConstantPoolCacheEntry::flags_offset())));
2632       __ lsr(c_rarg3, c_rarg3,
2633              ConstantPoolCacheEntry::tos_state_shift);
2634       ConstantPoolCacheEntry::verify_tos_state_shift();
2635       Label nope2, done, ok;
2636       __ ldr(c_rarg1, at_tos_p1());  // initially assume a one word jvalue
2637       __ cmpw(c_rarg3, ltos);
2638       __ br(Assembler::EQ, ok);
2639       __ cmpw(c_rarg3, dtos);
2640       __ br(Assembler::NE, nope2);
2641       __ bind(ok);
2642       __ ldr(c_rarg1, at_tos_p2()); // ltos (two word jvalue)
2643       __ bind(nope2);
2644     }
2645     // cache entry pointer
2646     __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2647     // object (tos)
2648     __ mov(c_rarg3, esp);
2649     // c_rarg1: object pointer set up above (NULL if static)
2650     // c_rarg2: cache entry pointer
2651     // c_rarg3: jvalue object on the stack
2652     __ call_VM(noreg,
2653                CAST_FROM_FN_PTR(address,
2654                                 InterpreterRuntime::post_field_modification),
2655                c_rarg1, c_rarg2, c_rarg3);
2656     __ get_cache_and_index_at_bcp(cache, index, 1);
2657     __ bind(L1);
2658   }
2659 }
2660 
2661 void TemplateTable::putfield_or_static(int byte_no, bool is_static) {
2662   transition(vtos, vtos);
2663 
2664   const Register cache = r2;
2665   const Register index = r3;
2666   const Register obj   = r2;
2667   const Register off   = r19;
2668   const Register flags = r0;
2669   const Register bc    = r4;
2670 
2671   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2672   jvmti_post_field_mod(cache, index, is_static);
2673   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2674 
2675   Label Done;
2676   __ mov(r5, flags);
2677 
2678   {
2679     Label notVolatile;
2680     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2681     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2682     __ bind(notVolatile);
2683   }
2684 
2685   // field address
2686   const Address field(obj, off);
2687 
2688   Label notByte, notBool, notInt, notShort, notChar,
2689         notLong, notFloat, notObj, notDouble;
2690 
2691   // x86 uses a shift and mask or wings it with a shift plus assert
2692   // the mask is not needed. aarch64 just uses bitfield extract
2693   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
2694 
2695   assert(btos == 0, "change code, btos != 0");
2696   __ cbnz(flags, notByte);
2697 
2698   // btos
2699   {
2700     __ pop(btos);
2701     if (!is_static) pop_and_check_object(obj);
2702     __ strb(r0, field);
2703     if (!is_static) {
2704       patch_bytecode(Bytecodes::_fast_bputfield, bc, r1, true, byte_no);
2705     }
2706     __ b(Done);
2707   }
2708 
2709   __ bind(notByte);
2710   __ cmp(flags, ztos);
2711   __ br(Assembler::NE, notBool);
2712 
2713   // ztos
2714   {
2715     __ pop(ztos);
2716     if (!is_static) pop_and_check_object(obj);
2717     __ andw(r0, r0, 0x1);
2718     __ strb(r0, field);
2719     if (!is_static) {
2720       patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
2721     }
2722     __ b(Done);
2723   }
2724 
2725   __ bind(notBool);
2726   __ cmp(flags, atos);
2727   __ br(Assembler::NE, notObj);
2728 
2729   // atos
2730   {
2731     __ pop(atos);
2732     if (!is_static) pop_and_check_object(obj);
2733     // Store into the field
2734     do_oop_store(_masm, field, r0, _bs->kind(), false);
2735     if (!is_static) {
2736       patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
2737     }
2738     __ b(Done);
2739   }
2740 
2741   __ bind(notObj);
2742   __ cmp(flags, itos);
2743   __ br(Assembler::NE, notInt);
2744 
2745   // itos
2746   {
2747     __ pop(itos);
2748     if (!is_static) pop_and_check_object(obj);
2749     __ strw(r0, field);
2750     if (!is_static) {
2751       patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
2752     }
2753     __ b(Done);
2754   }
2755 
2756   __ bind(notInt);
2757   __ cmp(flags, ctos);
2758   __ br(Assembler::NE, notChar);
2759 
2760   // ctos
2761   {
2762     __ pop(ctos);
2763     if (!is_static) pop_and_check_object(obj);
2764     __ strh(r0, field);
2765     if (!is_static) {
2766       patch_bytecode(Bytecodes::_fast_cputfield, bc, r1, true, byte_no);
2767     }
2768     __ b(Done);
2769   }
2770 
2771   __ bind(notChar);
2772   __ cmp(flags, stos);
2773   __ br(Assembler::NE, notShort);
2774 
2775   // stos
2776   {
2777     __ pop(stos);
2778     if (!is_static) pop_and_check_object(obj);
2779     __ strh(r0, field);
2780     if (!is_static) {
2781       patch_bytecode(Bytecodes::_fast_sputfield, bc, r1, true, byte_no);
2782     }
2783     __ b(Done);
2784   }
2785 
2786   __ bind(notShort);
2787   __ cmp(flags, ltos);
2788   __ br(Assembler::NE, notLong);
2789 
2790   // ltos
2791   {
2792     __ pop(ltos);
2793     if (!is_static) pop_and_check_object(obj);
2794     __ str(r0, field);
2795     if (!is_static) {
2796       patch_bytecode(Bytecodes::_fast_lputfield, bc, r1, true, byte_no);
2797     }
2798     __ b(Done);
2799   }
2800 
2801   __ bind(notLong);
2802   __ cmp(flags, ftos);
2803   __ br(Assembler::NE, notFloat);
2804 
2805   // ftos
2806   {
2807     __ pop(ftos);
2808     if (!is_static) pop_and_check_object(obj);
2809     __ strs(v0, field);
2810     if (!is_static) {
2811       patch_bytecode(Bytecodes::_fast_fputfield, bc, r1, true, byte_no);
2812     }
2813     __ b(Done);
2814   }
2815 
2816   __ bind(notFloat);
2817 #ifdef ASSERT
2818   __ cmp(flags, dtos);
2819   __ br(Assembler::NE, notDouble);
2820 #endif
2821 
2822   // dtos
2823   {
2824     __ pop(dtos);
2825     if (!is_static) pop_and_check_object(obj);
2826     __ strd(v0, field);
2827     if (!is_static) {
2828       patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
2829     }
2830   }
2831 
2832 #ifdef ASSERT
2833   __ b(Done);
2834 
2835   __ bind(notDouble);
2836   __ stop("Bad state");
2837 #endif
2838 
2839   __ bind(Done);
2840 
2841   {
2842     Label notVolatile;
2843     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2844     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
2845     __ bind(notVolatile);
2846   }
2847 }
2848 
2849 void TemplateTable::putfield(int byte_no)
2850 {
2851   putfield_or_static(byte_no, false);
2852 }
2853 
2854 void TemplateTable::putstatic(int byte_no) {
2855   putfield_or_static(byte_no, true);
2856 }
2857 
2858 void TemplateTable::jvmti_post_fast_field_mod()
2859 {
2860   if (JvmtiExport::can_post_field_modification()) {
2861     // Check to see if a field modification watch has been set before
2862     // we take the time to call into the VM.
2863     Label L2;
2864     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2865     __ ldrw(c_rarg3, Address(rscratch1));
2866     __ cbzw(c_rarg3, L2);
2867     __ pop_ptr(r19);                  // copy the object pointer from tos
2868     __ verify_oop(r19);
2869     __ push_ptr(r19);                 // put the object pointer back on tos
2870     // Save tos values before call_VM() clobbers them. Since we have
2871     // to do it for every data type, we use the saved values as the
2872     // jvalue object.
2873     switch (bytecode()) {          // load values into the jvalue object
2874     case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
2875     case Bytecodes::_fast_bputfield: // fall through
2876     case Bytecodes::_fast_zputfield: // fall through
2877     case Bytecodes::_fast_sputfield: // fall through
2878     case Bytecodes::_fast_cputfield: // fall through
2879     case Bytecodes::_fast_iputfield: __ push_i(r0); break;
2880     case Bytecodes::_fast_dputfield: __ push_d(); break;
2881     case Bytecodes::_fast_fputfield: __ push_f(); break;
2882     case Bytecodes::_fast_lputfield: __ push_l(r0); break;
2883 
2884     default:
2885       ShouldNotReachHere();
2886     }
2887     __ mov(c_rarg3, esp);             // points to jvalue on the stack
2888     // access constant pool cache entry
2889     __ get_cache_entry_pointer_at_bcp(c_rarg2, r0, 1);
2890     __ verify_oop(r19);
2891     // r19: object pointer copied above
2892     // c_rarg2: cache entry pointer
2893     // c_rarg3: jvalue object on the stack
2894     __ call_VM(noreg,
2895                CAST_FROM_FN_PTR(address,
2896                                 InterpreterRuntime::post_field_modification),
2897                r19, c_rarg2, c_rarg3);
2898 
2899     switch (bytecode()) {             // restore tos values
2900     case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
2901     case Bytecodes::_fast_bputfield: // fall through
2902     case Bytecodes::_fast_zputfield: // fall through
2903     case Bytecodes::_fast_sputfield: // fall through
2904     case Bytecodes::_fast_cputfield: // fall through
2905     case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
2906     case Bytecodes::_fast_dputfield: __ pop_d(); break;
2907     case Bytecodes::_fast_fputfield: __ pop_f(); break;
2908     case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
2909     }
2910     __ bind(L2);
2911   }
2912 }
2913 
2914 void TemplateTable::fast_storefield(TosState state)
2915 {
2916   transition(state, vtos);
2917 
2918   ByteSize base = ConstantPoolCache::base_offset();
2919 
2920   jvmti_post_fast_field_mod();
2921 
2922   // access constant pool cache
2923   __ get_cache_and_index_at_bcp(r2, r1, 1);
2924 
2925   // test for volatile with r3
2926   __ ldrw(r3, Address(r2, in_bytes(base +
2927                                    ConstantPoolCacheEntry::flags_offset())));
2928 
2929   // replace index with field offset from cache entry
2930   __ ldr(r1, Address(r2, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
2931 
2932   {
2933     Label notVolatile;
2934     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2935     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2936     __ bind(notVolatile);
2937   }
2938 
2939   Label notVolatile;
2940 
2941   // Get object from stack
2942   pop_and_check_object(r2);
2943 
2944   // field address
2945   const Address field(r2, r1);
2946 
2947   // access field
2948   switch (bytecode()) {
2949   case Bytecodes::_fast_aputfield:
2950     do_oop_store(_masm, field, r0, _bs->kind(), false);
2951     break;
2952   case Bytecodes::_fast_lputfield:
2953     __ str(r0, field);
2954     break;
2955   case Bytecodes::_fast_iputfield:
2956     __ strw(r0, field);
2957     break;
2958   case Bytecodes::_fast_zputfield:
2959     __ andw(r0, r0, 0x1);  // boolean is true if LSB is 1
2960     // fall through to bputfield
2961   case Bytecodes::_fast_bputfield:
2962     __ strb(r0, field);
2963     break;
2964   case Bytecodes::_fast_sputfield:
2965     // fall through
2966   case Bytecodes::_fast_cputfield:
2967     __ strh(r0, field);
2968     break;
2969   case Bytecodes::_fast_fputfield:
2970     __ strs(v0, field);
2971     break;
2972   case Bytecodes::_fast_dputfield:
2973     __ strd(v0, field);
2974     break;
2975   default:
2976     ShouldNotReachHere();
2977   }
2978 
2979   {
2980     Label notVolatile;
2981     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2982     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
2983     __ bind(notVolatile);
2984   }
2985 }
2986 
2987 
2988 void TemplateTable::fast_accessfield(TosState state)
2989 {
2990   transition(atos, state);
2991   // Do the JVMTI work here to avoid disturbing the register state below
2992   if (JvmtiExport::can_post_field_access()) {
2993     // Check to see if a field access watch has been set before we
2994     // take the time to call into the VM.
2995     Label L1;
2996     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2997     __ ldrw(r2, Address(rscratch1));
2998     __ cbzw(r2, L1);
2999     // access constant pool cache entry
3000     __ get_cache_entry_pointer_at_bcp(c_rarg2, rscratch2, 1);
3001     __ verify_oop(r0);
3002     __ push_ptr(r0);  // save object pointer before call_VM() clobbers it
3003     __ mov(c_rarg1, r0);
3004     // c_rarg1: object pointer copied above
3005     // c_rarg2: cache entry pointer
3006     __ call_VM(noreg,
3007                CAST_FROM_FN_PTR(address,
3008                                 InterpreterRuntime::post_field_access),
3009                c_rarg1, c_rarg2);
3010     __ pop_ptr(r0); // restore object pointer
3011     __ bind(L1);
3012   }
3013 
3014   // access constant pool cache
3015   __ get_cache_and_index_at_bcp(r2, r1, 1);
3016   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3017                                   ConstantPoolCacheEntry::f2_offset())));
3018   __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3019                                    ConstantPoolCacheEntry::flags_offset())));
3020 
3021   // r0: object
3022   __ verify_oop(r0);
3023   __ null_check(r0);
3024   const Address field(r0, r1);
3025 
3026   // 8179954: We need to make sure that the code generated for
3027   // volatile accesses forms a sequentially-consistent set of
3028   // operations when combined with STLR and LDAR.  Without a leading
3029   // membar it's possible for a simple Dekker test to fail if loads
3030   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3031   // the stores in one method and we interpret the loads in another.
3032   if (! UseBarriersForVolatile) {
3033     Label notVolatile;
3034     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3035     __ membar(MacroAssembler::AnyAny);
3036     __ bind(notVolatile);
3037   }
3038 
3039   // access field
3040   switch (bytecode()) {
3041   case Bytecodes::_fast_agetfield:
3042     __ load_heap_oop(r0, field);
3043     __ verify_oop(r0);
3044     break;
3045   case Bytecodes::_fast_lgetfield:
3046     __ ldr(r0, field);
3047     break;
3048   case Bytecodes::_fast_igetfield:
3049     __ ldrw(r0, field);
3050     break;
3051   case Bytecodes::_fast_bgetfield:
3052     __ load_signed_byte(r0, field);
3053     break;
3054   case Bytecodes::_fast_sgetfield:
3055     __ load_signed_short(r0, field);
3056     break;
3057   case Bytecodes::_fast_cgetfield:
3058     __ load_unsigned_short(r0, field);
3059     break;
3060   case Bytecodes::_fast_fgetfield:
3061     __ ldrs(v0, field);
3062     break;
3063   case Bytecodes::_fast_dgetfield:
3064     __ ldrd(v0, field);
3065     break;
3066   default:
3067     ShouldNotReachHere();
3068   }
3069   {
3070     Label notVolatile;
3071     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3072     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3073     __ bind(notVolatile);
3074   }
3075 }
3076 
3077 void TemplateTable::fast_xaccess(TosState state)
3078 {
3079   transition(vtos, state);
3080 
3081   // get receiver
3082   __ ldr(r0, aaddress(0));
3083   // access constant pool cache
3084   __ get_cache_and_index_at_bcp(r2, r3, 2);
3085   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3086                                   ConstantPoolCacheEntry::f2_offset())));
3087 
3088   // 8179954: We need to make sure that the code generated for
3089   // volatile accesses forms a sequentially-consistent set of
3090   // operations when combined with STLR and LDAR.  Without a leading
3091   // membar it's possible for a simple Dekker test to fail if loads
3092   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3093   // the stores in one method and we interpret the loads in another.
3094   if (! UseBarriersForVolatile) {
3095     Label notVolatile;
3096     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3097                                      ConstantPoolCacheEntry::flags_offset())));
3098     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3099     __ membar(MacroAssembler::AnyAny);
3100     __ bind(notVolatile);
3101   }
3102 
3103   // make sure exception is reported in correct bcp range (getfield is
3104   // next instruction)
3105   __ increment(rbcp);
3106   __ null_check(r0);
3107   switch (state) {
3108   case itos:
3109     __ ldrw(r0, Address(r0, r1, Address::lsl(0)));
3110     break;
3111   case atos:
3112     __ load_heap_oop(r0, Address(r0, r1, Address::lsl(0)));
3113     __ verify_oop(r0);
3114     break;
3115   case ftos:
3116     __ ldrs(v0, Address(r0, r1, Address::lsl(0)));
3117     break;
3118   default:
3119     ShouldNotReachHere();
3120   }
3121 
3122   {
3123     Label notVolatile;
3124     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3125                                      ConstantPoolCacheEntry::flags_offset())));
3126     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3127     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3128     __ bind(notVolatile);
3129   }
3130 
3131   __ decrement(rbcp);
3132 }
3133 
3134 
3135 
3136 //-----------------------------------------------------------------------------
3137 // Calls
3138 
3139 void TemplateTable::count_calls(Register method, Register temp)
3140 {
3141   __ call_Unimplemented();
3142 }
3143 
3144 void TemplateTable::prepare_invoke(int byte_no,
3145                                    Register method, // linked method (or i-klass)
3146                                    Register index,  // itable index, MethodType, etc.
3147                                    Register recv,   // if caller wants to see it
3148                                    Register flags   // if caller wants to test it
3149                                    ) {
3150   // determine flags
3151   Bytecodes::Code code = bytecode();
3152   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3153   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3154   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3155   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3156   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3157   const bool load_receiver       = (recv  != noreg);
3158   const bool save_flags          = (flags != noreg);
3159   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3160   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3161   assert(flags == noreg || flags == r3, "");
3162   assert(recv  == noreg || recv  == r2, "");
3163 
3164   // setup registers & access constant pool cache
3165   if (recv  == noreg)  recv  = r2;
3166   if (flags == noreg)  flags = r3;
3167   assert_different_registers(method, index, recv, flags);
3168 
3169   // save 'interpreter return address'
3170   __ save_bcp();
3171 
3172   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3173 
3174   // maybe push appendix to arguments (just before return address)
3175   if (is_invokedynamic || is_invokehandle) {
3176     Label L_no_push;
3177     __ tbz(flags, ConstantPoolCacheEntry::has_appendix_shift, L_no_push);
3178     // Push the appendix as a trailing parameter.
3179     // This must be done before we get the receiver,
3180     // since the parameter_size includes it.
3181     __ push(r19);
3182     __ mov(r19, index);
3183     assert(ConstantPoolCacheEntry::_indy_resolved_references_appendix_offset == 0, "appendix expected at index+0");
3184     __ load_resolved_reference_at_index(index, r19);
3185     __ pop(r19);
3186     __ push(index);  // push appendix (MethodType, CallSite, etc.)
3187     __ bind(L_no_push);
3188   }
3189 
3190   // load receiver if needed (note: no return address pushed yet)
3191   if (load_receiver) {
3192     __ andw(recv, flags, ConstantPoolCacheEntry::parameter_size_mask);
3193     // FIXME -- is this actually correct? looks like it should be 2
3194     // const int no_return_pc_pushed_yet = -1;  // argument slot correction before we push return address
3195     // const int receiver_is_at_end      = -1;  // back off one slot to get receiver
3196     // Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
3197     // __ movptr(recv, recv_addr);
3198     __ add(rscratch1, esp, recv, ext::uxtx, 3); // FIXME: uxtb here?
3199     __ ldr(recv, Address(rscratch1, -Interpreter::expr_offset_in_bytes(1)));
3200     __ verify_oop(recv);
3201   }
3202 
3203   // compute return type
3204   // x86 uses a shift and mask or wings it with a shift plus assert
3205   // the mask is not needed. aarch64 just uses bitfield extract
3206   __ ubfxw(rscratch2, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
3207   // load return address
3208   {
3209     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3210     __ mov(rscratch1, table_addr);
3211     __ ldr(lr, Address(rscratch1, rscratch2, Address::lsl(3)));
3212   }
3213 }
3214 
3215 
3216 void TemplateTable::invokevirtual_helper(Register index,
3217                                          Register recv,
3218                                          Register flags)
3219 {
3220   // Uses temporary registers r0, r3
3221   assert_different_registers(index, recv, r0, r3);
3222   // Test for an invoke of a final method
3223   Label notFinal;
3224   __ tbz(flags, ConstantPoolCacheEntry::is_vfinal_shift, notFinal);
3225 
3226   const Register method = index;  // method must be rmethod
3227   assert(method == rmethod,
3228          "methodOop must be rmethod for interpreter calling convention");
3229 
3230   // do the call - the index is actually the method to call
3231   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3232 
3233   // It's final, need a null check here!
3234   __ null_check(recv);
3235 
3236   // profile this call
3237   __ profile_final_call(r0);
3238   __ profile_arguments_type(r0, method, r4, true);
3239 
3240   __ jump_from_interpreted(method, r0);
3241 
3242   __ bind(notFinal);
3243 
3244   // get receiver klass
3245   __ null_check(recv, oopDesc::klass_offset_in_bytes());
3246   __ load_klass(r0, recv);
3247 
3248   // profile this call
3249   __ profile_virtual_call(r0, rlocals, r3);
3250 
3251   // get target methodOop & entry point
3252   __ lookup_virtual_method(r0, index, method);
3253   __ profile_arguments_type(r3, method, r4, true);
3254   // FIXME -- this looks completely redundant. is it?
3255   // __ ldr(r3, Address(method, Method::interpreter_entry_offset()));
3256   __ jump_from_interpreted(method, r3);
3257 }
3258 
3259 void TemplateTable::invokevirtual(int byte_no)
3260 {
3261   transition(vtos, vtos);
3262   assert(byte_no == f2_byte, "use this argument");
3263 
3264   prepare_invoke(byte_no, rmethod, noreg, r2, r3);
3265 
3266   // rmethod: index (actually a Method*)
3267   // r2: receiver
3268   // r3: flags
3269 
3270   invokevirtual_helper(rmethod, r2, r3);
3271 }
3272 
3273 void TemplateTable::invokespecial(int byte_no)
3274 {
3275   transition(vtos, vtos);
3276   assert(byte_no == f1_byte, "use this argument");
3277 
3278   prepare_invoke(byte_no, rmethod, noreg,  // get f1 Method*
3279                  r2);  // get receiver also for null check
3280   __ verify_oop(r2);
3281   __ null_check(r2);
3282   // do the call
3283   __ profile_call(r0);
3284   __ profile_arguments_type(r0, rmethod, rbcp, false);
3285   __ jump_from_interpreted(rmethod, r0);
3286 }
3287 
3288 void TemplateTable::invokestatic(int byte_no)
3289 {
3290   transition(vtos, vtos);
3291   assert(byte_no == f1_byte, "use this argument");
3292 
3293   prepare_invoke(byte_no, rmethod);  // get f1 Method*
3294   // do the call
3295   __ profile_call(r0);
3296   __ profile_arguments_type(r0, rmethod, r4, false);
3297   __ jump_from_interpreted(rmethod, r0);
3298 }
3299 
3300 void TemplateTable::fast_invokevfinal(int byte_no)
3301 {
3302   __ call_Unimplemented();
3303 }
3304 
3305 void TemplateTable::invokeinterface(int byte_no) {
3306   transition(vtos, vtos);
3307   assert(byte_no == f1_byte, "use this argument");
3308 
3309   prepare_invoke(byte_no, r0, rmethod,  // get f1 Klass*, f2 Method*
3310                  r2, r3); // recv, flags
3311 
3312   // r0: interface klass (from f1)
3313   // rmethod: method (from f2)
3314   // r2: receiver
3315   // r3: flags
3316 
3317   // Special case of invokeinterface called for virtual method of
3318   // java.lang.Object.  See cpCacheOop.cpp for details.
3319   // This code isn't produced by javac, but could be produced by
3320   // another compliant java compiler.
3321   Label notMethod;
3322   __ tbz(r3, ConstantPoolCacheEntry::is_forced_virtual_shift, notMethod);
3323 
3324   invokevirtual_helper(rmethod, r2, r3);
3325   __ bind(notMethod);
3326 
3327   // Get receiver klass into r3 - also a null check
3328   __ restore_locals();
3329   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3330   __ load_klass(r3, r2);
3331 
3332   Label no_such_interface, no_such_method;
3333 
3334   // Receiver subtype check against REFC.
3335   // Superklass in r0. Subklass in r3. Blows rscratch2, r13.
3336   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3337                              r3, r0, noreg,
3338                              // outputs: scan temp. reg, scan temp. reg
3339                              rscratch2, r13,
3340                              no_such_interface,
3341                              /*return_method=*/false);
3342 
3343   // profile this call
3344   __ profile_virtual_call(r3, r13, r19);
3345 
3346   // Get declaring interface class from method, and itable index
3347   __ ldr(r0, Address(rmethod, Method::const_offset()));
3348   __ ldr(r0, Address(r0, ConstMethod::constants_offset()));
3349   __ ldr(r0, Address(r0, ConstantPool::pool_holder_offset_in_bytes()));
3350   __ ldrw(rmethod, Address(rmethod, Method::itable_index_offset()));
3351   __ subw(rmethod, rmethod, Method::itable_index_max);
3352   __ negw(rmethod, rmethod);
3353 
3354   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3355                              r3, r0, rmethod,
3356                              // outputs: method, scan temp. reg
3357                              rmethod, r13,
3358                              no_such_interface);
3359 
3360   // rmethod,: methodOop to call
3361   // r2: receiver
3362   // Check for abstract method error
3363   // Note: This should be done more efficiently via a throw_abstract_method_error
3364   //       interpreter entry point and a conditional jump to it in case of a null
3365   //       method.
3366   __ cbz(rmethod, no_such_method);
3367 
3368   __ profile_arguments_type(r3, rmethod, r13, true);
3369 
3370   // do the call
3371   // r2: receiver
3372   // rmethod,: methodOop
3373   __ jump_from_interpreted(rmethod, r3);
3374   __ should_not_reach_here();
3375 
3376   // exception handling code follows...
3377   // note: must restore interpreter registers to canonical
3378   //       state for exception handling to work correctly!
3379 
3380   __ bind(no_such_method);
3381   // throw exception
3382   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3383   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3384   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
3385   // the call_VM checks for exception, so we should never return here.
3386   __ should_not_reach_here();
3387 
3388   __ bind(no_such_interface);
3389   // throw exception
3390   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3391   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3392   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3393                    InterpreterRuntime::throw_IncompatibleClassChangeError));
3394   // the call_VM checks for exception, so we should never return here.
3395   __ should_not_reach_here();
3396   return;
3397 }
3398 
3399 void TemplateTable::invokehandle(int byte_no) {
3400   transition(vtos, vtos);
3401   assert(byte_no == f1_byte, "use this argument");
3402 
3403   if (!EnableInvokeDynamic) {
3404     // rewriter does not generate this bytecode
3405     __ should_not_reach_here();
3406     return;
3407   }
3408 
3409   prepare_invoke(byte_no, rmethod, r0, r2);
3410   __ verify_method_ptr(r2);
3411   __ verify_oop(r2);
3412   __ null_check(r2);
3413 
3414   // FIXME: profile the LambdaForm also
3415 
3416   // r13 is safe to use here as a scratch reg because it is about to
3417   // be clobbered by jump_from_interpreted().
3418   __ profile_final_call(r13);
3419   __ profile_arguments_type(r13, rmethod, r4, true);
3420 
3421   __ jump_from_interpreted(rmethod, r0);
3422 }
3423 
3424 void TemplateTable::invokedynamic(int byte_no) {
3425   transition(vtos, vtos);
3426   assert(byte_no == f1_byte, "use this argument");
3427 
3428   if (!EnableInvokeDynamic) {
3429     // We should not encounter this bytecode if !EnableInvokeDynamic.
3430     // The verifier will stop it.  However, if we get past the verifier,
3431     // this will stop the thread in a reasonable way, without crashing the JVM.
3432     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3433                      InterpreterRuntime::throw_IncompatibleClassChangeError));
3434     // the call_VM checks for exception, so we should never return here.
3435     __ should_not_reach_here();
3436     return;
3437   }
3438 
3439   prepare_invoke(byte_no, rmethod, r0);
3440 
3441   // r0: CallSite object (from cpool->resolved_references[])
3442   // rmethod: MH.linkToCallSite method (from f2)
3443 
3444   // Note:  r0_callsite is already pushed by prepare_invoke
3445 
3446   // %%% should make a type profile for any invokedynamic that takes a ref argument
3447   // profile this call
3448   __ profile_call(rbcp);
3449   __ profile_arguments_type(r3, rmethod, r13, false);
3450 
3451   __ verify_oop(r0);
3452 
3453   __ jump_from_interpreted(rmethod, r0);
3454 }
3455 
3456 
3457 //-----------------------------------------------------------------------------
3458 // Allocation
3459 
3460 void TemplateTable::_new() {
3461   transition(vtos, atos);
3462 
3463   __ get_unsigned_2_byte_index_at_bcp(r3, 1);
3464   Label slow_case;
3465   Label done;
3466   Label initialize_header;
3467   Label initialize_object; // including clearing the fields
3468   Label allocate_shared;
3469 
3470   __ get_cpool_and_tags(r4, r0);
3471   // Make sure the class we're about to instantiate has been resolved.
3472   // This is done before loading InstanceKlass to be consistent with the order
3473   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3474   const int tags_offset = Array<u1>::base_offset_in_bytes();
3475   __ lea(rscratch1, Address(r0, r3, Address::lsl(0)));
3476   __ lea(rscratch1, Address(rscratch1, tags_offset));
3477   __ ldarb(rscratch1, rscratch1);
3478   __ cmp(rscratch1, JVM_CONSTANT_Class);
3479   __ br(Assembler::NE, slow_case);
3480 
3481   // get InstanceKlass
3482   __ lea(r4, Address(r4, r3, Address::lsl(3)));
3483   __ ldr(r4, Address(r4, sizeof(ConstantPool)));
3484 
3485   // make sure klass is initialized & doesn't have finalizer
3486   // make sure klass is fully initialized
3487   __ ldrb(rscratch1, Address(r4, InstanceKlass::init_state_offset()));
3488   __ cmp(rscratch1, InstanceKlass::fully_initialized);
3489   __ br(Assembler::NE, slow_case);
3490 
3491   // get instance_size in InstanceKlass (scaled to a count of bytes)
3492   __ ldrw(r3,
3493           Address(r4,
3494                   Klass::layout_helper_offset()));
3495   // test to see if it has a finalizer or is malformed in some way
3496   __ tbnz(r3, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
3497 
3498   // Allocate the instance
3499   // 1) Try to allocate in the TLAB
3500   // 2) if fail and the object is large allocate in the shared Eden
3501   // 3) if the above fails (or is not applicable), go to a slow case
3502   // (creates a new TLAB, etc.)
3503 
3504   const bool allow_shared_alloc =
3505     Universe::heap()->supports_inline_contig_alloc() && !CMSIncrementalMode;
3506 
3507   if (UseTLAB) {
3508     __ tlab_allocate(r0, r3, 0, noreg, r1,
3509                      allow_shared_alloc ? allocate_shared : slow_case);
3510 
3511     if (ZeroTLAB) {
3512       // the fields have been already cleared
3513       __ b(initialize_header);
3514     } else {
3515       // initialize both the header and fields
3516       __ b(initialize_object);
3517     }
3518   }
3519 
3520   // Allocation in the shared Eden, if allowed.
3521   //
3522   // r3: instance size in bytes
3523   if (allow_shared_alloc) {
3524     __ bind(allocate_shared);
3525 
3526     __ eden_allocate(r0, r3, 0, r10, slow_case);
3527     __ incr_allocated_bytes(rthread, r3, 0, rscratch1);
3528   }
3529 
3530   if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) {
3531     // The object is initialized before the header.  If the object size is
3532     // zero, go directly to the header initialization.
3533     __ bind(initialize_object);
3534     __ sub(r3, r3, sizeof(oopDesc));
3535     __ cbz(r3, initialize_header);
3536 
3537     // Initialize object fields
3538     {
3539       __ add(r2, r0, sizeof(oopDesc));
3540       Label loop;
3541       __ bind(loop);
3542       __ str(zr, Address(__ post(r2, BytesPerLong)));
3543       __ sub(r3, r3, BytesPerLong);
3544       __ cbnz(r3, loop);
3545     }
3546 
3547     // initialize object header only.
3548     __ bind(initialize_header);
3549     if (UseBiasedLocking) {
3550       __ ldr(rscratch1, Address(r4, Klass::prototype_header_offset()));
3551     } else {
3552       __ mov(rscratch1, (intptr_t)markOopDesc::prototype());
3553     }
3554     __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
3555     __ store_klass_gap(r0, zr);  // zero klass gap for compressed oops
3556     __ store_klass(r0, r4);      // store klass last
3557 
3558     {
3559       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3560       // Trigger dtrace event for fastpath
3561       __ push(atos); // save the return value
3562       __ call_VM_leaf(
3563            CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), r0);
3564       __ pop(atos); // restore the return value
3565 
3566     }
3567     __ b(done);
3568   }
3569 
3570   // slow case
3571   __ bind(slow_case);
3572   __ get_constant_pool(c_rarg1);
3573   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3574   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3575   __ verify_oop(r0);
3576 
3577   // continue
3578   __ bind(done);
3579   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3580   __ membar(Assembler::StoreStore);
3581 }
3582 
3583 void TemplateTable::newarray() {
3584   transition(itos, atos);
3585   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3586   __ mov(c_rarg2, r0);
3587   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3588           c_rarg1, c_rarg2);
3589   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3590   __ membar(Assembler::StoreStore);
3591 }
3592 
3593 void TemplateTable::anewarray() {
3594   transition(itos, atos);
3595   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3596   __ get_constant_pool(c_rarg1);
3597   __ mov(c_rarg3, r0);
3598   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3599           c_rarg1, c_rarg2, c_rarg3);
3600   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3601   __ membar(Assembler::StoreStore);
3602 }
3603 
3604 void TemplateTable::arraylength() {
3605   transition(atos, itos);
3606   __ null_check(r0, arrayOopDesc::length_offset_in_bytes());
3607   __ ldrw(r0, Address(r0, arrayOopDesc::length_offset_in_bytes()));
3608 }
3609 
3610 void TemplateTable::checkcast()
3611 {
3612   transition(atos, atos);
3613   Label done, is_null, ok_is_subtype, quicked, resolved;
3614   __ cbz(r0, is_null);
3615 
3616   // Get cpool & tags index
3617   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3618   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3619   // See if bytecode has already been quicked
3620   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3621   __ lea(r1, Address(rscratch1, r19));
3622   __ ldarb(r1, r1);
3623   __ cmp(r1, JVM_CONSTANT_Class);
3624   __ br(Assembler::EQ, quicked);
3625 
3626   __ push(atos); // save receiver for result, and for GC
3627   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3628   // vm_result_2 has metadata result
3629   __ get_vm_result_2(r0, rthread);
3630   __ pop(r3); // restore receiver
3631   __ b(resolved);
3632 
3633   // Get superklass in r0 and subklass in r3
3634   __ bind(quicked);
3635   __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
3636   __ lea(r0, Address(r2, r19, Address::lsl(3)));
3637   __ ldr(r0, Address(r0, sizeof(ConstantPool)));
3638 
3639   __ bind(resolved);
3640   __ load_klass(r19, r3);
3641 
3642   // Generate subtype check.  Blows r2, r5.  Object in r3.
3643   // Superklass in r0.  Subklass in r19.
3644   __ gen_subtype_check(r19, ok_is_subtype);
3645 
3646   // Come here on failure
3647   __ push(r3);
3648   // object is at TOS
3649   __ b(Interpreter::_throw_ClassCastException_entry);
3650 
3651   // Come here on success
3652   __ bind(ok_is_subtype);
3653   __ mov(r0, r3); // Restore object in r3
3654 
3655   // Collect counts on whether this test sees NULLs a lot or not.
3656   if (ProfileInterpreter) {
3657     __ b(done);
3658     __ bind(is_null);
3659     __ profile_null_seen(r2);
3660   } else {
3661     __ bind(is_null);   // same as 'done'
3662   }
3663   __ bind(done);
3664 }
3665 
3666 void TemplateTable::instanceof() {
3667   transition(atos, itos);
3668   Label done, is_null, ok_is_subtype, quicked, resolved;
3669   __ cbz(r0, is_null);
3670 
3671   // Get cpool & tags index
3672   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3673   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3674   // See if bytecode has already been quicked
3675   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3676   __ lea(r1, Address(rscratch1, r19));
3677   __ ldarb(r1, r1);
3678   __ cmp(r1, JVM_CONSTANT_Class);
3679   __ br(Assembler::EQ, quicked);
3680 
3681   __ push(atos); // save receiver for result, and for GC
3682   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3683   // vm_result_2 has metadata result
3684   __ get_vm_result_2(r0, rthread);
3685   __ pop(r3); // restore receiver
3686   __ verify_oop(r3);
3687   __ load_klass(r3, r3);
3688   __ b(resolved);
3689 
3690   // Get superklass in r0 and subklass in r3
3691   __ bind(quicked);
3692   __ load_klass(r3, r0);
3693   __ lea(r0, Address(r2, r19, Address::lsl(3)));
3694   __ ldr(r0, Address(r0, sizeof(ConstantPool)));
3695 
3696   __ bind(resolved);
3697 
3698   // Generate subtype check.  Blows r2, r5
3699   // Superklass in r0.  Subklass in r3.
3700   __ gen_subtype_check(r3, ok_is_subtype);
3701 
3702   // Come here on failure
3703   __ mov(r0, 0);
3704   __ b(done);
3705   // Come here on success
3706   __ bind(ok_is_subtype);
3707   __ mov(r0, 1);
3708 
3709   // Collect counts on whether this test sees NULLs a lot or not.
3710   if (ProfileInterpreter) {
3711     __ b(done);
3712     __ bind(is_null);
3713     __ profile_null_seen(r2);
3714   } else {
3715     __ bind(is_null);   // same as 'done'
3716   }
3717   __ bind(done);
3718   // r0 = 0: obj == NULL or  obj is not an instanceof the specified klass
3719   // r0 = 1: obj != NULL and obj is     an instanceof the specified klass
3720 }
3721 
3722 //-----------------------------------------------------------------------------
3723 // Breakpoints
3724 void TemplateTable::_breakpoint() {
3725   // Note: We get here even if we are single stepping..
3726   // jbug inists on setting breakpoints at every bytecode
3727   // even if we are in single step mode.
3728 
3729   transition(vtos, vtos);
3730 
3731   // get the unpatched byte code
3732   __ get_method(c_rarg1);
3733   __ call_VM(noreg,
3734              CAST_FROM_FN_PTR(address,
3735                               InterpreterRuntime::get_original_bytecode_at),
3736              c_rarg1, rbcp);
3737   __ mov(r19, r0);
3738 
3739   // post the breakpoint event
3740   __ call_VM(noreg,
3741              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
3742              rmethod, rbcp);
3743 
3744   // complete the execution of original bytecode
3745   __ mov(rscratch1, r19);
3746   __ dispatch_only_normal(vtos);
3747 }
3748 
3749 //-----------------------------------------------------------------------------
3750 // Exceptions
3751 
3752 void TemplateTable::athrow() {
3753   transition(atos, vtos);
3754   __ null_check(r0);
3755   __ b(Interpreter::throw_exception_entry());
3756 }
3757 
3758 //-----------------------------------------------------------------------------
3759 // Synchronization
3760 //
3761 // Note: monitorenter & exit are symmetric routines; which is reflected
3762 //       in the assembly code structure as well
3763 //
3764 // Stack layout:
3765 //
3766 // [expressions  ] <--- esp               = expression stack top
3767 // ..
3768 // [expressions  ]
3769 // [monitor entry] <--- monitor block top = expression stack bot
3770 // ..
3771 // [monitor entry]
3772 // [frame data   ] <--- monitor block bot
3773 // ...
3774 // [saved rbp    ] <--- rbp
3775 void TemplateTable::monitorenter()
3776 {
3777   transition(atos, vtos);
3778 
3779   // check for NULL object
3780   __ null_check(r0);
3781 
3782   const Address monitor_block_top(
3783         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3784   const Address monitor_block_bot(
3785         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3786   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3787 
3788   Label allocated;
3789 
3790   // initialize entry pointer
3791   __ mov(c_rarg1, zr); // points to free slot or NULL
3792 
3793   // find a free slot in the monitor block (result in c_rarg1)
3794   {
3795     Label entry, loop, exit;
3796     __ ldr(c_rarg3, monitor_block_top); // points to current entry,
3797                                         // starting with top-most entry
3798     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3799 
3800     __ b(entry);
3801 
3802     __ bind(loop);
3803     // check if current entry is used
3804     // if not used then remember entry in c_rarg1
3805     __ ldr(rscratch1, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
3806     __ cmp(zr, rscratch1);
3807     __ csel(c_rarg1, c_rarg3, c_rarg1, Assembler::EQ);
3808     // check if current entry is for same object
3809     __ cmp(r0, rscratch1);
3810     // if same object then stop searching
3811     __ br(Assembler::EQ, exit);
3812     // otherwise advance to next entry
3813     __ add(c_rarg3, c_rarg3, entry_size);
3814     __ bind(entry);
3815     // check if bottom reached
3816     __ cmp(c_rarg3, c_rarg2);
3817     // if not at bottom then check this entry
3818     __ br(Assembler::NE, loop);
3819     __ bind(exit);
3820   }
3821 
3822   __ cbnz(c_rarg1, allocated); // check if a slot has been found and
3823                             // if found, continue with that on
3824 
3825   // allocate one if there's no free slot
3826   {
3827     Label entry, loop;
3828     // 1. compute new pointers            // rsp: old expression stack top
3829     __ ldr(c_rarg1, monitor_block_bot);   // c_rarg1: old expression stack bottom
3830     __ sub(esp, esp, entry_size);         // move expression stack top
3831     __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
3832     __ mov(c_rarg3, esp);                 // set start value for copy loop
3833     __ str(c_rarg1, monitor_block_bot);   // set new monitor block bottom
3834 
3835     __ sub(sp, sp, entry_size);           // make room for the monitor
3836 
3837     __ b(entry);
3838     // 2. move expression stack contents
3839     __ bind(loop);
3840     __ ldr(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
3841                                                    // word from old location
3842     __ str(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
3843     __ add(c_rarg3, c_rarg3, wordSize);            // advance to next word
3844     __ bind(entry);
3845     __ cmp(c_rarg3, c_rarg1);        // check if bottom reached
3846     __ br(Assembler::NE, loop);      // if not at bottom then
3847                                      // copy next word
3848   }
3849 
3850   // call run-time routine
3851   // c_rarg1: points to monitor entry
3852   __ bind(allocated);
3853 
3854   // Increment bcp to point to the next bytecode, so exception
3855   // handling for async. exceptions work correctly.
3856   // The object has already been poped from the stack, so the
3857   // expression stack looks correct.
3858   __ increment(rbcp);
3859 
3860   // store object
3861   __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3862   __ lock_object(c_rarg1);
3863 
3864   // check to make sure this monitor doesn't cause stack overflow after locking
3865   __ save_bcp();  // in case of exception
3866   __ generate_stack_overflow_check(0);
3867 
3868   // The bcp has already been incremented. Just need to dispatch to
3869   // next instruction.
3870   __ dispatch_next(vtos);
3871 }
3872 
3873 
3874 void TemplateTable::monitorexit()
3875 {
3876   transition(atos, vtos);
3877 
3878   // check for NULL object
3879   __ null_check(r0);
3880 
3881   const Address monitor_block_top(
3882         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3883   const Address monitor_block_bot(
3884         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3885   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3886 
3887   Label found;
3888 
3889   // find matching slot
3890   {
3891     Label entry, loop;
3892     __ ldr(c_rarg1, monitor_block_top); // points to current entry,
3893                                         // starting with top-most entry
3894     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3895                                         // of monitor block
3896     __ b(entry);
3897 
3898     __ bind(loop);
3899     // check if current entry is for same object
3900     __ ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3901     __ cmp(r0, rscratch1);
3902     // if same object then stop searching
3903     __ br(Assembler::EQ, found);
3904     // otherwise advance to next entry
3905     __ add(c_rarg1, c_rarg1, entry_size);
3906     __ bind(entry);
3907     // check if bottom reached
3908     __ cmp(c_rarg1, c_rarg2);
3909     // if not at bottom then check this entry
3910     __ br(Assembler::NE, loop);
3911   }
3912 
3913   // error handling. Unlocking was not block-structured
3914   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3915                    InterpreterRuntime::throw_illegal_monitor_state_exception));
3916   __ should_not_reach_here();
3917 
3918   // call run-time routine
3919   __ bind(found);
3920   __ push_ptr(r0); // make sure object is on stack (contract with oopMaps)
3921   __ unlock_object(c_rarg1);
3922   __ pop_ptr(r0); // discard object
3923 }
3924 
3925 
3926 // Wide instructions
3927 void TemplateTable::wide()
3928 {
3929   __ load_unsigned_byte(r19, at_bcp(1));
3930   __ mov(rscratch1, (address)Interpreter::_wentry_point);
3931   __ ldr(rscratch1, Address(rscratch1, r19, Address::uxtw(3)));
3932   __ br(rscratch1);
3933 }
3934 
3935 
3936 // Multi arrays
3937 void TemplateTable::multianewarray() {
3938   transition(vtos, atos);
3939   __ load_unsigned_byte(r0, at_bcp(3)); // get number of dimensions
3940   // last dim is on top of stack; we want address of first one:
3941   // first_addr = last_addr + (ndims - 1) * wordSize
3942   __ lea(c_rarg1, Address(esp, r0, Address::uxtw(3)));
3943   __ sub(c_rarg1, c_rarg1, wordSize);
3944   call_VM(r0,
3945           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
3946           c_rarg1);
3947   __ load_unsigned_byte(r1, at_bcp(3));
3948   __ lea(esp, Address(esp, r1, Address::uxtw(3)));
3949 }
3950 #endif // !CC_INTERP