1 /*
   2  * Copyright (c) 2013, Red Hat Inc.
   3  * Copyright (c) 2003, 2011, Oracle and/or its affiliates.
   4  * All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "interpreter/interpreter.hpp"
  30 #include "interpreter/interpreterRuntime.hpp"
  31 #include "interpreter/templateTable.hpp"
  32 #include "memory/universe.inline.hpp"
  33 #include "oops/methodData.hpp"
  34 #include "oops/method.hpp"
  35 #include "oops/objArrayKlass.hpp"
  36 #include "oops/oop.inline.hpp"
  37 #include "prims/methodHandles.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/synchronizer.hpp"
  41 #if INCLUDE_ALL_GCS
  42 #include "shenandoahBarrierSetAssembler_aarch64.hpp"
  43 #endif
  44 
  45 #ifndef CC_INTERP
  46 
  47 #define __ _masm->
  48 
  49 // Platform-dependent initialization
  50 
  51 void TemplateTable::pd_initialize() {
  52   // No amd64 specific initialization
  53 }
  54 
  55 // Address computation: local variables
  56 
  57 static inline Address iaddress(int n) {
  58   return Address(rlocals, Interpreter::local_offset_in_bytes(n));
  59 }
  60 
  61 static inline Address laddress(int n) {
  62   return iaddress(n + 1);
  63 }
  64 
  65 static inline Address faddress(int n) {
  66   return iaddress(n);
  67 }
  68 
  69 static inline Address daddress(int n) {
  70   return laddress(n);
  71 }
  72 
  73 static inline Address aaddress(int n) {
  74   return iaddress(n);
  75 }
  76 
  77 static inline Address iaddress(Register r) {
  78   return Address(rlocals, r, Address::lsl(3));
  79 }
  80 
  81 static inline Address laddress(Register r, Register scratch,
  82                                InterpreterMacroAssembler* _masm) {
  83   __ lea(scratch, Address(rlocals, r, Address::lsl(3)));
  84   return Address(scratch, Interpreter::local_offset_in_bytes(1));
  85 }
  86 
  87 static inline Address faddress(Register r) {
  88   return iaddress(r);
  89 }
  90 
  91 static inline Address daddress(Register r, Register scratch,
  92                                InterpreterMacroAssembler* _masm) {
  93   return laddress(r, scratch, _masm);
  94 }
  95 
  96 static inline Address aaddress(Register r) {
  97   return iaddress(r);
  98 }
  99 
 100 static inline Address at_rsp() {
 101   return Address(esp, 0);
 102 }
 103 
 104 // At top of Java expression stack which may be different than esp().  It
 105 // isn't for category 1 objects.
 106 static inline Address at_tos   () {
 107   return Address(esp,  Interpreter::expr_offset_in_bytes(0));
 108 }
 109 
 110 static inline Address at_tos_p1() {
 111   return Address(esp,  Interpreter::expr_offset_in_bytes(1));
 112 }
 113 
 114 static inline Address at_tos_p2() {
 115   return Address(esp,  Interpreter::expr_offset_in_bytes(2));
 116 }
 117 
 118 static inline Address at_tos_p3() {
 119   return Address(esp,  Interpreter::expr_offset_in_bytes(3));
 120 }
 121 
 122 static inline Address at_tos_p4() {
 123   return Address(esp,  Interpreter::expr_offset_in_bytes(4));
 124 }
 125 
 126 static inline Address at_tos_p5() {
 127   return Address(esp,  Interpreter::expr_offset_in_bytes(5));
 128 }
 129 
 130 // Condition conversion
 131 static Assembler::Condition j_not(TemplateTable::Condition cc) {
 132   switch (cc) {
 133   case TemplateTable::equal        : return Assembler::NE;
 134   case TemplateTable::not_equal    : return Assembler::EQ;
 135   case TemplateTable::less         : return Assembler::GE;
 136   case TemplateTable::less_equal   : return Assembler::GT;
 137   case TemplateTable::greater      : return Assembler::LE;
 138   case TemplateTable::greater_equal: return Assembler::LT;
 139   }
 140   ShouldNotReachHere();
 141   return Assembler::EQ;
 142 }
 143 
 144 
 145 // Miscelaneous helper routines
 146 // Store an oop (or NULL) at the Address described by obj.
 147 // If val == noreg this means store a NULL
 148 static void do_oop_store(InterpreterMacroAssembler* _masm,
 149                          Address obj,
 150                          Register val,
 151                          BarrierSet::Name barrier,
 152                          bool precise) {
 153   assert(val == noreg || val == r0, "parameter is just for looks");
 154   switch (barrier) {
 155 #if INCLUDE_ALL_GCS
 156     case BarrierSet::G1SATBCT:
 157     case BarrierSet::G1SATBCTLogging:
 158       {
 159         // flatten object address if needed
 160         if (obj.index() == noreg && obj.offset() == 0) {
 161           if (obj.base() != r3) {
 162             __ mov(r3, obj.base());
 163           }
 164         } else {
 165           __ lea(r3, obj);
 166         }
 167         __ g1_write_barrier_pre(r3 /* obj */,
 168                                 r1 /* pre_val */,
 169                                 rthread /* thread */,
 170                                 r10  /* tmp */,
 171                                 val != noreg /* tosca_live */,
 172                                 false /* expand_call */);
 173         if (val == noreg) {
 174           __ store_heap_oop_null(Address(r3, 0));
 175         } else {
 176           // G1 barrier needs uncompressed oop for region cross check.
 177           Register new_val = val;
 178           if (UseCompressedOops) {
 179             new_val = rscratch2;
 180             __ mov(new_val, val);
 181           }
 182 
 183           __ store_heap_oop(Address(r3, 0), val);
 184 
 185           __ g1_write_barrier_post(r3 /* store_adr */,
 186                                    new_val /* new_val */,
 187                                    rthread /* thread */,
 188                                    r10 /* tmp */,
 189                                    r1 /* tmp2 */);
 190         }
 191 
 192       }
 193       break;
 194     case BarrierSet::ShenandoahBarrierSet:
 195       {
 196         // flatten object address if needed
 197         if (obj.index() == noreg && obj.offset() == 0) {
 198           if (obj.base() != r3) {
 199             __ mov(r3, obj.base());
 200           }
 201         } else {
 202           __ lea(r3, obj);
 203         }
 204         if (ShenandoahSATBBarrier) {
 205           __ g1_write_barrier_pre(r3 /* obj */,
 206                                   r1 /* pre_val */,
 207                                   rthread /* thread */,
 208                                   r10  /* tmp */,
 209                                   val != noreg /* tosca_live */,
 210                                   false /* expand_call */);
 211         }
 212         if (val == noreg) {
 213           __ store_heap_oop_null(Address(r3, 0));
 214         } else {
 215           if (ShenandoahStoreValEnqueueBarrier) {
 216             ShenandoahBarrierSetAssembler::bsasm()->storeval_barrier(_masm, val, r10);
 217           }
 218           __ store_heap_oop(Address(r3, 0), val);
 219         }
 220 
 221       }
 222       break;
 223 #endif // INCLUDE_ALL_GCS
 224     case BarrierSet::CardTableModRef:
 225     case BarrierSet::CardTableExtension:
 226       {
 227         if (val == noreg) {
 228           __ store_heap_oop_null(obj);
 229         } else {
 230           __ store_heap_oop(obj, val);
 231           // flatten object address if needed
 232           if (!precise || (obj.index() == noreg && obj.offset() == 0)) {
 233             __ store_check(obj.base());
 234           } else {
 235             __ lea(r3, obj);
 236             __ store_check(r3);
 237           }
 238         }
 239       }
 240       break;
 241     case BarrierSet::ModRef:
 242     case BarrierSet::Other:
 243       if (val == noreg) {
 244         __ store_heap_oop_null(obj);
 245       } else {
 246         __ store_heap_oop(obj, val);
 247       }
 248       break;
 249     default      :
 250       ShouldNotReachHere();
 251 
 252   }
 253 }
 254 
 255 Address TemplateTable::at_bcp(int offset) {
 256   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 257   return Address(rbcp, offset);
 258 }
 259 
 260 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 261                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 262                                    int byte_no)
 263 {
 264   if (!RewriteBytecodes)  return;
 265   Label L_patch_done;
 266 
 267   switch (bc) {
 268   case Bytecodes::_fast_aputfield:
 269   case Bytecodes::_fast_bputfield:
 270   case Bytecodes::_fast_zputfield:
 271   case Bytecodes::_fast_cputfield:
 272   case Bytecodes::_fast_dputfield:
 273   case Bytecodes::_fast_fputfield:
 274   case Bytecodes::_fast_iputfield:
 275   case Bytecodes::_fast_lputfield:
 276   case Bytecodes::_fast_sputfield:
 277     {
 278       // We skip bytecode quickening for putfield instructions when
 279       // the put_code written to the constant pool cache is zero.
 280       // This is required so that every execution of this instruction
 281       // calls out to InterpreterRuntime::resolve_get_put to do
 282       // additional, required work.
 283       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 284       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 285       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 286       __ movw(bc_reg, bc);
 287       __ cmpw(temp_reg, (unsigned) 0);
 288       __ br(Assembler::EQ, L_patch_done);  // don't patch
 289     }
 290     break;
 291   default:
 292     assert(byte_no == -1, "sanity");
 293     // the pair bytecodes have already done the load.
 294     if (load_bc_into_bc_reg) {
 295       __ movw(bc_reg, bc);
 296     }
 297   }
 298 
 299   if (JvmtiExport::can_post_breakpoint()) {
 300     Label L_fast_patch;
 301     // if a breakpoint is present we can't rewrite the stream directly
 302     __ load_unsigned_byte(temp_reg, at_bcp(0));
 303     __ cmpw(temp_reg, Bytecodes::_breakpoint);
 304     __ br(Assembler::NE, L_fast_patch);
 305     // Let breakpoint table handling rewrite to quicker bytecode
 306     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), rmethod, rbcp, bc_reg);
 307     __ b(L_patch_done);
 308     __ bind(L_fast_patch);
 309   }
 310 
 311 #ifdef ASSERT
 312   Label L_okay;
 313   __ load_unsigned_byte(temp_reg, at_bcp(0));
 314   __ cmpw(temp_reg, (int) Bytecodes::java_code(bc));
 315   __ br(Assembler::EQ, L_okay);
 316   __ cmpw(temp_reg, bc_reg);
 317   __ br(Assembler::EQ, L_okay);
 318   __ stop("patching the wrong bytecode");
 319   __ bind(L_okay);
 320 #endif
 321 
 322   // patch bytecode
 323   __ strb(bc_reg, at_bcp(0));
 324   __ bind(L_patch_done);
 325 }
 326 
 327 
 328 // Individual instructions
 329 
 330 void TemplateTable::nop() {
 331   transition(vtos, vtos);
 332   // nothing to do
 333 }
 334 
 335 void TemplateTable::shouldnotreachhere() {
 336   transition(vtos, vtos);
 337   __ stop("shouldnotreachhere bytecode");
 338 }
 339 
 340 void TemplateTable::aconst_null()
 341 {
 342   transition(vtos, atos);
 343   __ mov(r0, 0);
 344 }
 345 
 346 void TemplateTable::iconst(int value)
 347 {
 348   transition(vtos, itos);
 349   __ mov(r0, value);
 350 }
 351 
 352 void TemplateTable::lconst(int value)
 353 {
 354   __ mov(r0, value);
 355 }
 356 
 357 void TemplateTable::fconst(int value)
 358 {
 359   transition(vtos, ftos);
 360   switch (value) {
 361   case 0:
 362     __ fmovs(v0, zr);
 363     break;
 364   case 1:
 365     __ fmovs(v0, 1.0);
 366     break;
 367   case 2:
 368     __ fmovs(v0, 2.0);
 369     break;
 370   default:
 371     ShouldNotReachHere();
 372     break;
 373   }
 374 }
 375 
 376 void TemplateTable::dconst(int value)
 377 {
 378   transition(vtos, dtos);
 379   switch (value) {
 380   case 0:
 381     __ fmovd(v0, zr);
 382     break;
 383   case 1:
 384     __ fmovd(v0, 1.0);
 385     break;
 386   case 2:
 387     __ fmovd(v0, 2.0);
 388     break;
 389   default:
 390     ShouldNotReachHere();
 391     break;
 392   }
 393 }
 394 
 395 void TemplateTable::bipush()
 396 {
 397   transition(vtos, itos);
 398   __ load_signed_byte32(r0, at_bcp(1));
 399 }
 400 
 401 void TemplateTable::sipush()
 402 {
 403   transition(vtos, itos);
 404   __ load_unsigned_short(r0, at_bcp(1));
 405   __ revw(r0, r0);
 406   __ asrw(r0, r0, 16);
 407 }
 408 
 409 void TemplateTable::ldc(bool wide)
 410 {
 411   transition(vtos, vtos);
 412   Label call_ldc, notFloat, notClass, Done;
 413 
 414   if (wide) {
 415     __ get_unsigned_2_byte_index_at_bcp(r1, 1);
 416   } else {
 417     __ load_unsigned_byte(r1, at_bcp(1));
 418   }
 419   __ get_cpool_and_tags(r2, r0);
 420 
 421   const int base_offset = ConstantPool::header_size() * wordSize;
 422   const int tags_offset = Array<u1>::base_offset_in_bytes();
 423 
 424   // get type
 425   __ add(r3, r1, tags_offset);
 426   __ lea(r3, Address(r0, r3));
 427   __ ldarb(r3, r3);
 428 
 429   // unresolved class - get the resolved class
 430   __ cmp(r3, JVM_CONSTANT_UnresolvedClass);
 431   __ br(Assembler::EQ, call_ldc);
 432 
 433   // unresolved class in error state - call into runtime to throw the error
 434   // from the first resolution attempt
 435   __ cmp(r3, JVM_CONSTANT_UnresolvedClassInError);
 436   __ br(Assembler::EQ, call_ldc);
 437 
 438   // resolved class - need to call vm to get java mirror of the class
 439   __ cmp(r3, JVM_CONSTANT_Class);
 440   __ br(Assembler::NE, notClass);
 441 
 442   __ bind(call_ldc);
 443   __ mov(c_rarg1, wide);
 444   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 445   __ push_ptr(r0);
 446   __ verify_oop(r0);
 447   __ b(Done);
 448 
 449   __ bind(notClass);
 450   __ cmp(r3, JVM_CONSTANT_Float);
 451   __ br(Assembler::NE, notFloat);
 452   // ftos
 453   __ adds(r1, r2, r1, Assembler::LSL, 3);
 454   __ ldrs(v0, Address(r1, base_offset));
 455   __ push_f();
 456   __ b(Done);
 457 
 458   __ bind(notFloat);
 459 #ifdef ASSERT
 460   {
 461     Label L;
 462     __ cmp(r3, JVM_CONSTANT_Integer);
 463     __ br(Assembler::EQ, L);
 464     // String and Object are rewritten to fast_aldc
 465     __ stop("unexpected tag type in ldc");
 466     __ bind(L);
 467   }
 468 #endif
 469   // itos JVM_CONSTANT_Integer only
 470   __ adds(r1, r2, r1, Assembler::LSL, 3);
 471   __ ldrw(r0, Address(r1, base_offset));
 472   __ push_i(r0);
 473   __ bind(Done);
 474 }
 475 
 476 // Fast path for caching oop constants.
 477 void TemplateTable::fast_aldc(bool wide)
 478 {
 479   transition(vtos, atos);
 480 
 481   Register result = r0;
 482   Register tmp = r1;
 483   int index_size = wide ? sizeof(u2) : sizeof(u1);
 484 
 485   Label resolved;
 486 
 487   // We are resolved if the resolved reference cache entry contains a
 488   // non-null object (String, MethodType, etc.)
 489   assert_different_registers(result, tmp);
 490   __ get_cache_index_at_bcp(tmp, 1, index_size);
 491   __ load_resolved_reference_at_index(result, tmp);
 492   __ cbnz(result, resolved);
 493 
 494   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 495 
 496   // first time invocation - must resolve first
 497   __ mov(tmp, (int)bytecode());
 498   __ call_VM(result, entry, tmp);
 499 
 500   __ bind(resolved);
 501 
 502   if (VerifyOops) {
 503     __ verify_oop(result);
 504   }
 505 }
 506 
 507 void TemplateTable::ldc2_w()
 508 {
 509   transition(vtos, vtos);
 510   Label Long, Done;
 511   __ get_unsigned_2_byte_index_at_bcp(r0, 1);
 512 
 513   __ get_cpool_and_tags(r1, r2);
 514   const int base_offset = ConstantPool::header_size() * wordSize;
 515   const int tags_offset = Array<u1>::base_offset_in_bytes();
 516 
 517   // get type
 518   __ lea(r2, Address(r2, r0, Address::lsl(0)));
 519   __ load_unsigned_byte(r2, Address(r2, tags_offset));
 520   __ cmpw(r2, (int)JVM_CONSTANT_Double);
 521   __ br(Assembler::NE, Long);
 522   // dtos
 523   __ lea (r2, Address(r1, r0, Address::lsl(3)));
 524   __ ldrd(v0, Address(r2, base_offset));
 525   __ push_d();
 526   __ b(Done);
 527 
 528   __ bind(Long);
 529   // ltos
 530   __ lea(r0, Address(r1, r0, Address::lsl(3)));
 531   __ ldr(r0, Address(r0, base_offset));
 532   __ push_l();
 533 
 534   __ bind(Done);
 535 }
 536 
 537 void TemplateTable::locals_index(Register reg, int offset)
 538 {
 539   __ ldrb(reg, at_bcp(offset));
 540   __ neg(reg, reg);
 541 }
 542 
 543 void TemplateTable::iload()
 544 {
 545   transition(vtos, itos);
 546   if (RewriteFrequentPairs) {
 547     Label rewrite, done;
 548     Register bc = r4;
 549 
 550     // get next bytecode
 551     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 552 
 553     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 554     // last two iloads in a pair.  Comparing against fast_iload means that
 555     // the next bytecode is neither an iload or a caload, and therefore
 556     // an iload pair.
 557     __ cmpw(r1, Bytecodes::_iload);
 558     __ br(Assembler::EQ, done);
 559 
 560     // if _fast_iload rewrite to _fast_iload2
 561     __ cmpw(r1, Bytecodes::_fast_iload);
 562     __ movw(bc, Bytecodes::_fast_iload2);
 563     __ br(Assembler::EQ, rewrite);
 564 
 565     // if _caload rewrite to _fast_icaload
 566     __ cmpw(r1, Bytecodes::_caload);
 567     __ movw(bc, Bytecodes::_fast_icaload);
 568     __ br(Assembler::EQ, rewrite);
 569 
 570     // else rewrite to _fast_iload
 571     __ movw(bc, Bytecodes::_fast_iload);
 572 
 573     // rewrite
 574     // bc: new bytecode
 575     __ bind(rewrite);
 576     patch_bytecode(Bytecodes::_iload, bc, r1, false);
 577     __ bind(done);
 578 
 579   }
 580 
 581   // do iload, get the local value into tos
 582   locals_index(r1);
 583   __ ldr(r0, iaddress(r1));
 584 
 585 }
 586 
 587 void TemplateTable::fast_iload2()
 588 {
 589   transition(vtos, itos);
 590   locals_index(r1);
 591   __ ldr(r0, iaddress(r1));
 592   __ push(itos);
 593   locals_index(r1, 3);
 594   __ ldr(r0, iaddress(r1));
 595 }
 596 
 597 void TemplateTable::fast_iload()
 598 {
 599   transition(vtos, itos);
 600   locals_index(r1);
 601   __ ldr(r0, iaddress(r1));
 602 }
 603 
 604 void TemplateTable::lload()
 605 {
 606   transition(vtos, ltos);
 607   __ ldrb(r1, at_bcp(1));
 608   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 609   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 610 }
 611 
 612 void TemplateTable::fload()
 613 {
 614   transition(vtos, ftos);
 615   locals_index(r1);
 616   // n.b. we use ldrd here because this is a 64 bit slot
 617   // this is comparable to the iload case
 618   __ ldrd(v0, faddress(r1));
 619 }
 620 
 621 void TemplateTable::dload()
 622 {
 623   transition(vtos, dtos);
 624   __ ldrb(r1, at_bcp(1));
 625   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 626   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 627 }
 628 
 629 void TemplateTable::aload()
 630 {
 631   transition(vtos, atos);
 632   locals_index(r1);
 633   __ ldr(r0, iaddress(r1));
 634 }
 635 
 636 void TemplateTable::locals_index_wide(Register reg) {
 637   __ ldrh(reg, at_bcp(2));
 638   __ rev16w(reg, reg);
 639   __ neg(reg, reg);
 640 }
 641 
 642 void TemplateTable::wide_iload() {
 643   transition(vtos, itos);
 644   locals_index_wide(r1);
 645   __ ldr(r0, iaddress(r1));
 646 }
 647 
 648 void TemplateTable::wide_lload()
 649 {
 650   transition(vtos, ltos);
 651   __ ldrh(r1, at_bcp(2));
 652   __ rev16w(r1, r1);
 653   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 654   __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 655 }
 656 
 657 void TemplateTable::wide_fload()
 658 {
 659   transition(vtos, ftos);
 660   locals_index_wide(r1);
 661   // n.b. we use ldrd here because this is a 64 bit slot
 662   // this is comparable to the iload case
 663   __ ldrd(v0, faddress(r1));
 664 }
 665 
 666 void TemplateTable::wide_dload()
 667 {
 668   transition(vtos, dtos);
 669   __ ldrh(r1, at_bcp(2));
 670   __ rev16w(r1, r1);
 671   __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
 672   __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 673 }
 674 
 675 void TemplateTable::wide_aload()
 676 {
 677   transition(vtos, atos);
 678   locals_index_wide(r1);
 679   __ ldr(r0, aaddress(r1));
 680 }
 681 
 682 void TemplateTable::index_check(Register array, Register index)
 683 {
 684   // destroys r1, rscratch1
 685   // check array
 686   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
 687   // sign extend index for use by indexed load
 688   // __ movl2ptr(index, index);
 689   // check index
 690   Register length = rscratch1;
 691   __ ldrw(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 692   __ cmpw(index, length);
 693   if (index != r1) {
 694     // ??? convention: move aberrant index into r1 for exception message
 695     assert(r1 != array, "different registers");
 696     __ mov(r1, index);
 697   }
 698   Label ok;
 699   __ br(Assembler::LO, ok);
 700   __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 701   __ br(rscratch1);
 702   __ bind(ok);
 703 }
 704 
 705 void TemplateTable::iaload()
 706 {
 707   transition(itos, itos);
 708   __ mov(r1, r0);
 709   __ pop_ptr(r0);
 710   // r0: array
 711   // r1: index
 712   index_check(r0, r1); // leaves index in r1, kills rscratch1
 713   __ lea(r1, Address(r0, r1, Address::uxtw(2)));
 714   __ ldrw(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_INT)));
 715 }
 716 
 717 void TemplateTable::laload()
 718 {
 719   transition(itos, ltos);
 720   __ mov(r1, r0);
 721   __ pop_ptr(r0);
 722   // r0: array
 723   // r1: index
 724   index_check(r0, r1); // leaves index in r1, kills rscratch1
 725   __ lea(r1, Address(r0, r1, Address::uxtw(3)));
 726   __ ldr(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_LONG)));
 727 }
 728 
 729 void TemplateTable::faload()
 730 {
 731   transition(itos, ftos);
 732   __ mov(r1, r0);
 733   __ pop_ptr(r0);
 734   // r0: array
 735   // r1: index
 736   index_check(r0, r1); // leaves index in r1, kills rscratch1
 737   __ lea(r1,  Address(r0, r1, Address::uxtw(2)));
 738   __ ldrs(v0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
 739 }
 740 
 741 void TemplateTable::daload()
 742 {
 743   transition(itos, dtos);
 744   __ mov(r1, r0);
 745   __ pop_ptr(r0);
 746   // r0: array
 747   // r1: index
 748   index_check(r0, r1); // leaves index in r1, kills rscratch1
 749   __ lea(r1,  Address(r0, r1, Address::uxtw(3)));
 750   __ ldrd(v0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
 751 }
 752 
 753 void TemplateTable::aaload()
 754 {
 755   transition(itos, atos);
 756   __ mov(r1, r0);
 757   __ pop_ptr(r0);
 758   // r0: array
 759   // r1: index
 760   index_check(r0, r1); // leaves index in r1, kills rscratch1
 761   int s = (UseCompressedOops ? 2 : 3);
 762   __ lea(r1, Address(r0, r1, Address::uxtw(s)));
 763   __ load_heap_oop(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
 764 }
 765 
 766 void TemplateTable::baload()
 767 {
 768   transition(itos, itos);
 769   __ mov(r1, r0);
 770   __ pop_ptr(r0);
 771   // r0: array
 772   // r1: index
 773   index_check(r0, r1); // leaves index in r1, kills rscratch1
 774   __ lea(r1,  Address(r0, r1, Address::uxtw(0)));
 775   __ load_signed_byte(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_BYTE)));
 776 }
 777 
 778 void TemplateTable::caload()
 779 {
 780   transition(itos, itos);
 781   __ mov(r1, r0);
 782   __ pop_ptr(r0);
 783   // r0: array
 784   // r1: index
 785   index_check(r0, r1); // leaves index in r1, kills rscratch1
 786   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 787   __ load_unsigned_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 788 }
 789 
 790 // iload followed by caload frequent pair
 791 void TemplateTable::fast_icaload()
 792 {
 793   transition(vtos, itos);
 794   // load index out of locals
 795   locals_index(r2);
 796   __ ldr(r1, iaddress(r2));
 797 
 798   __ pop_ptr(r0);
 799 
 800   // r0: array
 801   // r1: index
 802   index_check(r0, r1); // leaves index in r1, kills rscratch1
 803   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 804   __ load_unsigned_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 805 }
 806 
 807 void TemplateTable::saload()
 808 {
 809   transition(itos, itos);
 810   __ mov(r1, r0);
 811   __ pop_ptr(r0);
 812   // r0: array
 813   // r1: index
 814   index_check(r0, r1); // leaves index in r1, kills rscratch1
 815   __ lea(r1,  Address(r0, r1, Address::uxtw(1)));
 816   __ load_signed_short(r0, Address(r1,  arrayOopDesc::base_offset_in_bytes(T_SHORT)));
 817 }
 818 
 819 void TemplateTable::iload(int n)
 820 {
 821   transition(vtos, itos);
 822   __ ldr(r0, iaddress(n));
 823 }
 824 
 825 void TemplateTable::lload(int n)
 826 {
 827   transition(vtos, ltos);
 828   __ ldr(r0, laddress(n));
 829 }
 830 
 831 void TemplateTable::fload(int n)
 832 {
 833   transition(vtos, ftos);
 834   __ ldrs(v0, faddress(n));
 835 }
 836 
 837 void TemplateTable::dload(int n)
 838 {
 839   transition(vtos, dtos);
 840   __ ldrd(v0, daddress(n));
 841 }
 842 
 843 void TemplateTable::aload(int n)
 844 {
 845   transition(vtos, atos);
 846   __ ldr(r0, iaddress(n));
 847 }
 848 
 849 void TemplateTable::aload_0()
 850 {
 851   // According to bytecode histograms, the pairs:
 852   //
 853   // _aload_0, _fast_igetfield
 854   // _aload_0, _fast_agetfield
 855   // _aload_0, _fast_fgetfield
 856   //
 857   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 858   // _aload_0 bytecode checks if the next bytecode is either
 859   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 860   // rewrites the current bytecode into a pair bytecode; otherwise it
 861   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 862   // the pair check anymore.
 863   //
 864   // Note: If the next bytecode is _getfield, the rewrite must be
 865   //       delayed, otherwise we may miss an opportunity for a pair.
 866   //
 867   // Also rewrite frequent pairs
 868   //   aload_0, aload_1
 869   //   aload_0, iload_1
 870   // These bytecodes with a small amount of code are most profitable
 871   // to rewrite
 872   if (RewriteFrequentPairs) {
 873     Label rewrite, done;
 874     const Register bc = r4;
 875 
 876     // get next bytecode
 877     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 878 
 879     // do actual aload_0
 880     aload(0);
 881 
 882     // if _getfield then wait with rewrite
 883     __ cmpw(r1, Bytecodes::Bytecodes::_getfield);
 884     __ br(Assembler::EQ, done);
 885 
 886     // if _igetfield then reqrite to _fast_iaccess_0
 887     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 888     __ cmpw(r1, Bytecodes::_fast_igetfield);
 889     __ movw(bc, Bytecodes::_fast_iaccess_0);
 890     __ br(Assembler::EQ, rewrite);
 891 
 892     // if _agetfield then reqrite to _fast_aaccess_0
 893     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 894     __ cmpw(r1, Bytecodes::_fast_agetfield);
 895     __ movw(bc, Bytecodes::_fast_aaccess_0);
 896     __ br(Assembler::EQ, rewrite);
 897 
 898     // if _fgetfield then reqrite to _fast_faccess_0
 899     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 900     __ cmpw(r1, Bytecodes::_fast_fgetfield);
 901     __ movw(bc, Bytecodes::_fast_faccess_0);
 902     __ br(Assembler::EQ, rewrite);
 903 
 904     // else rewrite to _fast_aload0
 905     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 906     __ movw(bc, Bytecodes::Bytecodes::_fast_aload_0);
 907 
 908     // rewrite
 909     // bc: new bytecode
 910     __ bind(rewrite);
 911     patch_bytecode(Bytecodes::_aload_0, bc, r1, false);
 912 
 913     __ bind(done);
 914   } else {
 915     aload(0);
 916   }
 917 }
 918 
 919 void TemplateTable::istore()
 920 {
 921   transition(itos, vtos);
 922   locals_index(r1);
 923   // FIXME: We're being very pernickerty here storing a jint in a
 924   // local with strw, which costs an extra instruction over what we'd
 925   // be able to do with a simple str.  We should just store the whole
 926   // word.
 927   __ lea(rscratch1, iaddress(r1));
 928   __ strw(r0, Address(rscratch1));
 929 }
 930 
 931 void TemplateTable::lstore()
 932 {
 933   transition(ltos, vtos);
 934   locals_index(r1);
 935   __ str(r0, laddress(r1, rscratch1, _masm));
 936 }
 937 
 938 void TemplateTable::fstore() {
 939   transition(ftos, vtos);
 940   locals_index(r1);
 941   __ lea(rscratch1, iaddress(r1));
 942   __ strs(v0, Address(rscratch1));
 943 }
 944 
 945 void TemplateTable::dstore() {
 946   transition(dtos, vtos);
 947   locals_index(r1);
 948   __ strd(v0, daddress(r1, rscratch1, _masm));
 949 }
 950 
 951 void TemplateTable::astore()
 952 {
 953   transition(vtos, vtos);
 954   __ pop_ptr(r0);
 955   locals_index(r1);
 956   __ str(r0, aaddress(r1));
 957 }
 958 
 959 void TemplateTable::wide_istore() {
 960   transition(vtos, vtos);
 961   __ pop_i();
 962   locals_index_wide(r1);
 963   __ lea(rscratch1, iaddress(r1));
 964   __ strw(r0, Address(rscratch1));
 965 }
 966 
 967 void TemplateTable::wide_lstore() {
 968   transition(vtos, vtos);
 969   __ pop_l();
 970   locals_index_wide(r1);
 971   __ str(r0, laddress(r1, rscratch1, _masm));
 972 }
 973 
 974 void TemplateTable::wide_fstore() {
 975   transition(vtos, vtos);
 976   __ pop_f();
 977   locals_index_wide(r1);
 978   __ lea(rscratch1, faddress(r1));
 979   __ strs(v0, rscratch1);
 980 }
 981 
 982 void TemplateTable::wide_dstore() {
 983   transition(vtos, vtos);
 984   __ pop_d();
 985   locals_index_wide(r1);
 986   __ strd(v0, daddress(r1, rscratch1, _masm));
 987 }
 988 
 989 void TemplateTable::wide_astore() {
 990   transition(vtos, vtos);
 991   __ pop_ptr(r0);
 992   locals_index_wide(r1);
 993   __ str(r0, aaddress(r1));
 994 }
 995 
 996 void TemplateTable::iastore() {
 997   transition(itos, vtos);
 998   __ pop_i(r1);
 999   __ pop_ptr(r3);
1000   // r0: value
1001   // r1: index
1002   // r3: array
1003   index_check(r3, r1); // prefer index in r1
1004   __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
1005   __ strw(r0, Address(rscratch1,
1006                       arrayOopDesc::base_offset_in_bytes(T_INT)));
1007 }
1008 
1009 void TemplateTable::lastore() {
1010   transition(ltos, vtos);
1011   __ pop_i(r1);
1012   __ pop_ptr(r3);
1013   // r0: value
1014   // r1: index
1015   // r3: array
1016   index_check(r3, r1); // prefer index in r1
1017   __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
1018   __ str(r0, Address(rscratch1,
1019                       arrayOopDesc::base_offset_in_bytes(T_LONG)));
1020 }
1021 
1022 void TemplateTable::fastore() {
1023   transition(ftos, vtos);
1024   __ pop_i(r1);
1025   __ pop_ptr(r3);
1026   // v0: value
1027   // r1:  index
1028   // r3:  array
1029   index_check(r3, r1); // prefer index in r1
1030   __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
1031   __ strs(v0, Address(rscratch1,
1032                       arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
1033 }
1034 
1035 void TemplateTable::dastore() {
1036   transition(dtos, vtos);
1037   __ pop_i(r1);
1038   __ pop_ptr(r3);
1039   // v0: value
1040   // r1:  index
1041   // r3:  array
1042   index_check(r3, r1); // prefer index in r1
1043   __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
1044   __ strd(v0, Address(rscratch1,
1045                       arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
1046 }
1047 
1048 void TemplateTable::aastore() {
1049   Label is_null, ok_is_subtype, done;
1050   transition(vtos, vtos);
1051   // stack: ..., array, index, value
1052   __ ldr(r0, at_tos());    // value
1053   __ ldr(r2, at_tos_p1()); // index
1054   __ ldr(r3, at_tos_p2()); // array
1055 
1056   Address element_address(r4, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
1057 
1058   index_check(r3, r2);     // kills r1
1059   __ lea(r4, Address(r3, r2, Address::uxtw(UseCompressedOops? 2 : 3)));
1060 
1061   // do array store check - check for NULL value first
1062   __ cbz(r0, is_null);
1063 
1064   // Move subklass into r1
1065   __ load_klass(r1, r0);
1066   // Move superklass into r0
1067   __ load_klass(r0, r3);
1068   __ ldr(r0, Address(r0,
1069                      ObjArrayKlass::element_klass_offset()));
1070   // Compress array + index*oopSize + 12 into a single register.  Frees r2.
1071 
1072   // Generate subtype check.  Blows r2, r5
1073   // Superklass in r0.  Subklass in r1.
1074   __ gen_subtype_check(r1, ok_is_subtype);
1075 
1076   // Come here on failure
1077   // object is at TOS
1078   __ b(Interpreter::_throw_ArrayStoreException_entry);
1079 
1080   // Come here on success
1081   __ bind(ok_is_subtype);
1082 
1083   // Get the value we will store
1084   __ ldr(r0, at_tos());
1085   // Now store using the appropriate barrier
1086   do_oop_store(_masm, element_address, r0, _bs->kind(), true);
1087   __ b(done);
1088 
1089   // Have a NULL in r0, r3=array, r2=index.  Store NULL at ary[idx]
1090   __ bind(is_null);
1091   __ profile_null_seen(r2);
1092 
1093   // Store a NULL
1094   do_oop_store(_masm, element_address, noreg, _bs->kind(), true);
1095 
1096   // Pop stack arguments
1097   __ bind(done);
1098   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1099 }
1100 
1101 void TemplateTable::bastore()
1102 {
1103   transition(itos, vtos);
1104   __ pop_i(r1);
1105   __ pop_ptr(r3);
1106   // r0: value
1107   // r1: index
1108   // r3: array
1109   index_check(r3, r1); // prefer index in r1
1110 
1111   // Need to check whether array is boolean or byte
1112   // since both types share the bastore bytecode.
1113   __ load_klass(r2, r3);
1114   __ ldrw(r2, Address(r2, Klass::layout_helper_offset()));
1115   int diffbit = Klass::layout_helper_boolean_diffbit();
1116   __ andw(rscratch1, r2, diffbit);
1117   Label L_skip;
1118   __ cbzw(rscratch1, L_skip);
1119   __ andw(r0, r0, 1);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1120   __ bind(L_skip);
1121 
1122   __ lea(rscratch1, Address(r3, r1, Address::uxtw(0)));
1123   __ strb(r0, Address(rscratch1,
1124                       arrayOopDesc::base_offset_in_bytes(T_BYTE)));
1125 }
1126 
1127 void TemplateTable::castore()
1128 {
1129   transition(itos, vtos);
1130   __ pop_i(r1);
1131   __ pop_ptr(r3);
1132   // r0: value
1133   // r1: index
1134   // r3: array
1135   index_check(r3, r1); // prefer index in r1
1136   __ lea(rscratch1, Address(r3, r1, Address::uxtw(1)));
1137   __ strh(r0, Address(rscratch1,
1138                       arrayOopDesc::base_offset_in_bytes(T_CHAR)));
1139 }
1140 
1141 void TemplateTable::sastore()
1142 {
1143   castore();
1144 }
1145 
1146 void TemplateTable::istore(int n)
1147 {
1148   transition(itos, vtos);
1149   __ str(r0, iaddress(n));
1150 }
1151 
1152 void TemplateTable::lstore(int n)
1153 {
1154   transition(ltos, vtos);
1155   __ str(r0, laddress(n));
1156 }
1157 
1158 void TemplateTable::fstore(int n)
1159 {
1160   transition(ftos, vtos);
1161   __ strs(v0, faddress(n));
1162 }
1163 
1164 void TemplateTable::dstore(int n)
1165 {
1166   transition(dtos, vtos);
1167   __ strd(v0, daddress(n));
1168 }
1169 
1170 void TemplateTable::astore(int n)
1171 {
1172   transition(vtos, vtos);
1173   __ pop_ptr(r0);
1174   __ str(r0, iaddress(n));
1175 }
1176 
1177 void TemplateTable::pop()
1178 {
1179   transition(vtos, vtos);
1180   __ add(esp, esp, Interpreter::stackElementSize);
1181 }
1182 
1183 void TemplateTable::pop2()
1184 {
1185   transition(vtos, vtos);
1186   __ add(esp, esp, 2 * Interpreter::stackElementSize);
1187 }
1188 
1189 void TemplateTable::dup()
1190 {
1191   transition(vtos, vtos);
1192   __ ldr(r0, Address(esp, 0));
1193   __ push(r0);
1194   // stack: ..., a, a
1195 }
1196 
1197 void TemplateTable::dup_x1()
1198 {
1199   transition(vtos, vtos);
1200   // stack: ..., a, b
1201   __ ldr(r0, at_tos());  // load b
1202   __ ldr(r2, at_tos_p1());  // load a
1203   __ str(r0, at_tos_p1());  // store b
1204   __ str(r2, at_tos());  // store a
1205   __ push(r0);                  // push b
1206   // stack: ..., b, a, b
1207 }
1208 
1209 void TemplateTable::dup_x2()
1210 {
1211   transition(vtos, vtos);
1212   // stack: ..., a, b, c
1213   __ ldr(r0, at_tos());  // load c
1214   __ ldr(r2, at_tos_p2());  // load a
1215   __ str(r0, at_tos_p2());  // store c in a
1216   __ push(r0);      // push c
1217   // stack: ..., c, b, c, c
1218   __ ldr(r0, at_tos_p2());  // load b
1219   __ str(r2, at_tos_p2());  // store a in b
1220   // stack: ..., c, a, c, c
1221   __ str(r0, at_tos_p1());  // store b in c
1222   // stack: ..., c, a, b, c
1223 }
1224 
1225 void TemplateTable::dup2()
1226 {
1227   transition(vtos, vtos);
1228   // stack: ..., a, b
1229   __ ldr(r0, at_tos_p1());  // load a
1230   __ push(r0);                  // push a
1231   __ ldr(r0, at_tos_p1());  // load b
1232   __ push(r0);                  // push b
1233   // stack: ..., a, b, a, b
1234 }
1235 
1236 void TemplateTable::dup2_x1()
1237 {
1238   transition(vtos, vtos);
1239   // stack: ..., a, b, c
1240   __ ldr(r2, at_tos());  // load c
1241   __ ldr(r0, at_tos_p1());  // load b
1242   __ push(r0);                  // push b
1243   __ push(r2);                  // push c
1244   // stack: ..., a, b, c, b, c
1245   __ str(r2, at_tos_p3());  // store c in b
1246   // stack: ..., a, c, c, b, c
1247   __ ldr(r2, at_tos_p4());  // load a
1248   __ str(r2, at_tos_p2());  // store a in 2nd c
1249   // stack: ..., a, c, a, b, c
1250   __ str(r0, at_tos_p4());  // store b in a
1251   // stack: ..., b, c, a, b, c
1252 }
1253 
1254 void TemplateTable::dup2_x2()
1255 {
1256   transition(vtos, vtos);
1257   // stack: ..., a, b, c, d
1258   __ ldr(r2, at_tos());  // load d
1259   __ ldr(r0, at_tos_p1());  // load c
1260   __ push(r0)            ;      // push c
1261   __ push(r2);                  // push d
1262   // stack: ..., a, b, c, d, c, d
1263   __ ldr(r0, at_tos_p4());  // load b
1264   __ str(r0, at_tos_p2());  // store b in d
1265   __ str(r2, at_tos_p4());  // store d in b
1266   // stack: ..., a, d, c, b, c, d
1267   __ ldr(r2, at_tos_p5());  // load a
1268   __ ldr(r0, at_tos_p3());  // load c
1269   __ str(r2, at_tos_p3());  // store a in c
1270   __ str(r0, at_tos_p5());  // store c in a
1271   // stack: ..., c, d, a, b, c, d
1272 }
1273 
1274 void TemplateTable::swap()
1275 {
1276   transition(vtos, vtos);
1277   // stack: ..., a, b
1278   __ ldr(r2, at_tos_p1());  // load a
1279   __ ldr(r0, at_tos());  // load b
1280   __ str(r2, at_tos());  // store a in b
1281   __ str(r0, at_tos_p1());  // store b in a
1282   // stack: ..., b, a
1283 }
1284 
1285 void TemplateTable::iop2(Operation op)
1286 {
1287   transition(itos, itos);
1288   // r0 <== r1 op r0
1289   __ pop_i(r1);
1290   switch (op) {
1291   case add  : __ addw(r0, r1, r0); break;
1292   case sub  : __ subw(r0, r1, r0); break;
1293   case mul  : __ mulw(r0, r1, r0); break;
1294   case _and : __ andw(r0, r1, r0); break;
1295   case _or  : __ orrw(r0, r1, r0); break;
1296   case _xor : __ eorw(r0, r1, r0); break;
1297   case shl  : __ lslvw(r0, r1, r0); break;
1298   case shr  : __ asrvw(r0, r1, r0); break;
1299   case ushr : __ lsrvw(r0, r1, r0);break;
1300   default   : ShouldNotReachHere();
1301   }
1302 }
1303 
1304 void TemplateTable::lop2(Operation op)
1305 {
1306   transition(ltos, ltos);
1307   // r0 <== r1 op r0
1308   __ pop_l(r1);
1309   switch (op) {
1310   case add  : __ add(r0, r1, r0); break;
1311   case sub  : __ sub(r0, r1, r0); break;
1312   case mul  : __ mul(r0, r1, r0); break;
1313   case _and : __ andr(r0, r1, r0); break;
1314   case _or  : __ orr(r0, r1, r0); break;
1315   case _xor : __ eor(r0, r1, r0); break;
1316   default   : ShouldNotReachHere();
1317   }
1318 }
1319 
1320 void TemplateTable::idiv()
1321 {
1322   transition(itos, itos);
1323   // explicitly check for div0
1324   Label no_div0;
1325   __ cbnzw(r0, no_div0);
1326   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1327   __ br(rscratch1);
1328   __ bind(no_div0);
1329   __ pop_i(r1);
1330   // r0 <== r1 idiv r0
1331   __ corrected_idivl(r0, r1, r0, /* want_remainder */ false);
1332 }
1333 
1334 void TemplateTable::irem()
1335 {
1336   transition(itos, itos);
1337   // explicitly check for div0
1338   Label no_div0;
1339   __ cbnzw(r0, no_div0);
1340   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1341   __ br(rscratch1);
1342   __ bind(no_div0);
1343   __ pop_i(r1);
1344   // r0 <== r1 irem r0
1345   __ corrected_idivl(r0, r1, r0, /* want_remainder */ true);
1346 }
1347 
1348 void TemplateTable::lmul()
1349 {
1350   transition(ltos, ltos);
1351   __ pop_l(r1);
1352   __ mul(r0, r0, r1);
1353 }
1354 
1355 void TemplateTable::ldiv()
1356 {
1357   transition(ltos, ltos);
1358   // explicitly check for div0
1359   Label no_div0;
1360   __ cbnz(r0, no_div0);
1361   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1362   __ br(rscratch1);
1363   __ bind(no_div0);
1364   __ pop_l(r1);
1365   // r0 <== r1 ldiv r0
1366   __ corrected_idivq(r0, r1, r0, /* want_remainder */ false);
1367 }
1368 
1369 void TemplateTable::lrem()
1370 {
1371   transition(ltos, ltos);
1372   // explicitly check for div0
1373   Label no_div0;
1374   __ cbnz(r0, no_div0);
1375   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1376   __ br(rscratch1);
1377   __ bind(no_div0);
1378   __ pop_l(r1);
1379   // r0 <== r1 lrem r0
1380   __ corrected_idivq(r0, r1, r0, /* want_remainder */ true);
1381 }
1382 
1383 void TemplateTable::lshl()
1384 {
1385   transition(itos, ltos);
1386   // shift count is in r0
1387   __ pop_l(r1);
1388   __ lslv(r0, r1, r0);
1389 }
1390 
1391 void TemplateTable::lshr()
1392 {
1393   transition(itos, ltos);
1394   // shift count is in r0
1395   __ pop_l(r1);
1396   __ asrv(r0, r1, r0);
1397 }
1398 
1399 void TemplateTable::lushr()
1400 {
1401   transition(itos, ltos);
1402   // shift count is in r0
1403   __ pop_l(r1);
1404   __ lsrv(r0, r1, r0);
1405 }
1406 
1407 void TemplateTable::fop2(Operation op)
1408 {
1409   transition(ftos, ftos);
1410   switch (op) {
1411   case add:
1412     // n.b. use ldrd because this is a 64 bit slot
1413     __ pop_f(v1);
1414     __ fadds(v0, v1, v0);
1415     break;
1416   case sub:
1417     __ pop_f(v1);
1418     __ fsubs(v0, v1, v0);
1419     break;
1420   case mul:
1421     __ pop_f(v1);
1422     __ fmuls(v0, v1, v0);
1423     break;
1424   case div:
1425     __ pop_f(v1);
1426     __ fdivs(v0, v1, v0);
1427     break;
1428   case rem:
1429     __ fmovs(v1, v0);
1430     __ pop_f(v0);
1431     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem));
1432     break;
1433   default:
1434     ShouldNotReachHere();
1435     break;
1436   }
1437 }
1438 
1439 void TemplateTable::dop2(Operation op)
1440 {
1441   transition(dtos, dtos);
1442   switch (op) {
1443   case add:
1444     // n.b. use ldrd because this is a 64 bit slot
1445     __ pop_d(v1);
1446     __ faddd(v0, v1, v0);
1447     break;
1448   case sub:
1449     __ pop_d(v1);
1450     __ fsubd(v0, v1, v0);
1451     break;
1452   case mul:
1453     __ pop_d(v1);
1454     __ fmuld(v0, v1, v0);
1455     break;
1456   case div:
1457     __ pop_d(v1);
1458     __ fdivd(v0, v1, v0);
1459     break;
1460   case rem:
1461     __ fmovd(v1, v0);
1462     __ pop_d(v0);
1463     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem));
1464     break;
1465   default:
1466     ShouldNotReachHere();
1467     break;
1468   }
1469 }
1470 
1471 void TemplateTable::ineg()
1472 {
1473   transition(itos, itos);
1474   __ negw(r0, r0);
1475 
1476 }
1477 
1478 void TemplateTable::lneg()
1479 {
1480   transition(ltos, ltos);
1481   __ neg(r0, r0);
1482 }
1483 
1484 void TemplateTable::fneg()
1485 {
1486   transition(ftos, ftos);
1487   __ fnegs(v0, v0);
1488 }
1489 
1490 void TemplateTable::dneg()
1491 {
1492   transition(dtos, dtos);
1493   __ fnegd(v0, v0);
1494 }
1495 
1496 void TemplateTable::iinc()
1497 {
1498   transition(vtos, vtos);
1499   __ load_signed_byte(r1, at_bcp(2)); // get constant
1500   locals_index(r2);
1501   __ ldr(r0, iaddress(r2));
1502   __ addw(r0, r0, r1);
1503   __ str(r0, iaddress(r2));
1504 }
1505 
1506 void TemplateTable::wide_iinc()
1507 {
1508   transition(vtos, vtos);
1509   // __ mov(r1, zr);
1510   __ ldrw(r1, at_bcp(2)); // get constant and index
1511   __ rev16(r1, r1);
1512   __ ubfx(r2, r1, 0, 16);
1513   __ neg(r2, r2);
1514   __ sbfx(r1, r1, 16, 16);
1515   __ ldr(r0, iaddress(r2));
1516   __ addw(r0, r0, r1);
1517   __ str(r0, iaddress(r2));
1518 }
1519 
1520 void TemplateTable::convert()
1521 {
1522   // Checking
1523 #ifdef ASSERT
1524   {
1525     TosState tos_in  = ilgl;
1526     TosState tos_out = ilgl;
1527     switch (bytecode()) {
1528     case Bytecodes::_i2l: // fall through
1529     case Bytecodes::_i2f: // fall through
1530     case Bytecodes::_i2d: // fall through
1531     case Bytecodes::_i2b: // fall through
1532     case Bytecodes::_i2c: // fall through
1533     case Bytecodes::_i2s: tos_in = itos; break;
1534     case Bytecodes::_l2i: // fall through
1535     case Bytecodes::_l2f: // fall through
1536     case Bytecodes::_l2d: tos_in = ltos; break;
1537     case Bytecodes::_f2i: // fall through
1538     case Bytecodes::_f2l: // fall through
1539     case Bytecodes::_f2d: tos_in = ftos; break;
1540     case Bytecodes::_d2i: // fall through
1541     case Bytecodes::_d2l: // fall through
1542     case Bytecodes::_d2f: tos_in = dtos; break;
1543     default             : ShouldNotReachHere();
1544     }
1545     switch (bytecode()) {
1546     case Bytecodes::_l2i: // fall through
1547     case Bytecodes::_f2i: // fall through
1548     case Bytecodes::_d2i: // fall through
1549     case Bytecodes::_i2b: // fall through
1550     case Bytecodes::_i2c: // fall through
1551     case Bytecodes::_i2s: tos_out = itos; break;
1552     case Bytecodes::_i2l: // fall through
1553     case Bytecodes::_f2l: // fall through
1554     case Bytecodes::_d2l: tos_out = ltos; break;
1555     case Bytecodes::_i2f: // fall through
1556     case Bytecodes::_l2f: // fall through
1557     case Bytecodes::_d2f: tos_out = ftos; break;
1558     case Bytecodes::_i2d: // fall through
1559     case Bytecodes::_l2d: // fall through
1560     case Bytecodes::_f2d: tos_out = dtos; break;
1561     default             : ShouldNotReachHere();
1562     }
1563     transition(tos_in, tos_out);
1564   }
1565 #endif // ASSERT
1566   // static const int64_t is_nan = 0x8000000000000000L;
1567 
1568   // Conversion
1569   switch (bytecode()) {
1570   case Bytecodes::_i2l:
1571     __ sxtw(r0, r0);
1572     break;
1573   case Bytecodes::_i2f:
1574     __ scvtfws(v0, r0);
1575     break;
1576   case Bytecodes::_i2d:
1577     __ scvtfwd(v0, r0);
1578     break;
1579   case Bytecodes::_i2b:
1580     __ sxtbw(r0, r0);
1581     break;
1582   case Bytecodes::_i2c:
1583     __ uxthw(r0, r0);
1584     break;
1585   case Bytecodes::_i2s:
1586     __ sxthw(r0, r0);
1587     break;
1588   case Bytecodes::_l2i:
1589     __ uxtw(r0, r0);
1590     break;
1591   case Bytecodes::_l2f:
1592     __ scvtfs(v0, r0);
1593     break;
1594   case Bytecodes::_l2d:
1595     __ scvtfd(v0, r0);
1596     break;
1597   case Bytecodes::_f2i:
1598   {
1599     Label L_Okay;
1600     __ clear_fpsr();
1601     __ fcvtzsw(r0, v0);
1602     __ get_fpsr(r1);
1603     __ cbzw(r1, L_Okay);
1604     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2i));
1605     __ bind(L_Okay);
1606   }
1607     break;
1608   case Bytecodes::_f2l:
1609   {
1610     Label L_Okay;
1611     __ clear_fpsr();
1612     __ fcvtzs(r0, v0);
1613     __ get_fpsr(r1);
1614     __ cbzw(r1, L_Okay);
1615     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2l));
1616     __ bind(L_Okay);
1617   }
1618     break;
1619   case Bytecodes::_f2d:
1620     __ fcvts(v0, v0);
1621     break;
1622   case Bytecodes::_d2i:
1623   {
1624     Label L_Okay;
1625     __ clear_fpsr();
1626     __ fcvtzdw(r0, v0);
1627     __ get_fpsr(r1);
1628     __ cbzw(r1, L_Okay);
1629     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
1630     __ bind(L_Okay);
1631   }
1632     break;
1633   case Bytecodes::_d2l:
1634   {
1635     Label L_Okay;
1636     __ clear_fpsr();
1637     __ fcvtzd(r0, v0);
1638     __ get_fpsr(r1);
1639     __ cbzw(r1, L_Okay);
1640     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
1641     __ bind(L_Okay);
1642   }
1643     break;
1644   case Bytecodes::_d2f:
1645     __ fcvtd(v0, v0);
1646     break;
1647   default:
1648     ShouldNotReachHere();
1649   }
1650 }
1651 
1652 void TemplateTable::lcmp()
1653 {
1654   transition(ltos, itos);
1655   Label done;
1656   __ pop_l(r1);
1657   __ cmp(r1, r0);
1658   __ mov(r0, (u_int64_t)-1L);
1659   __ br(Assembler::LT, done);
1660   // __ mov(r0, 1UL);
1661   // __ csel(r0, r0, zr, Assembler::NE);
1662   // and here is a faster way
1663   __ csinc(r0, zr, zr, Assembler::EQ);
1664   __ bind(done);
1665 }
1666 
1667 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1668 {
1669   Label done;
1670   if (is_float) {
1671     // XXX get rid of pop here, use ... reg, mem32
1672     __ pop_f(v1);
1673     __ fcmps(v1, v0);
1674   } else {
1675     // XXX get rid of pop here, use ... reg, mem64
1676     __ pop_d(v1);
1677     __ fcmpd(v1, v0);
1678   }
1679   if (unordered_result < 0) {
1680     // we want -1 for unordered or less than, 0 for equal and 1 for
1681     // greater than.
1682     __ mov(r0, (u_int64_t)-1L);
1683     // for FP LT tests less than or unordered
1684     __ br(Assembler::LT, done);
1685     // install 0 for EQ otherwise 1
1686     __ csinc(r0, zr, zr, Assembler::EQ);
1687   } else {
1688     // we want -1 for less than, 0 for equal and 1 for unordered or
1689     // greater than.
1690     __ mov(r0, 1L);
1691     // for FP HI tests greater than or unordered
1692     __ br(Assembler::HI, done);
1693     // install 0 for EQ otherwise ~0
1694     __ csinv(r0, zr, zr, Assembler::EQ);
1695 
1696   }
1697   __ bind(done);
1698 }
1699 
1700 void TemplateTable::branch(bool is_jsr, bool is_wide)
1701 {
1702   // We might be moving to a safepoint.  The thread which calls
1703   // Interpreter::notice_safepoints() will effectively flush its cache
1704   // when it makes a system call, but we need to do something to
1705   // ensure that we see the changed dispatch table.
1706   __ membar(MacroAssembler::LoadLoad);
1707 
1708   __ profile_taken_branch(r0, r1);
1709   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1710                              InvocationCounter::counter_offset();
1711   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1712                               InvocationCounter::counter_offset();
1713 
1714   // load branch displacement
1715   if (!is_wide) {
1716     __ ldrh(r2, at_bcp(1));
1717     __ rev16(r2, r2);
1718     // sign extend the 16 bit value in r2
1719     __ sbfm(r2, r2, 0, 15);
1720   } else {
1721     __ ldrw(r2, at_bcp(1));
1722     __ revw(r2, r2);
1723     // sign extend the 32 bit value in r2
1724     __ sbfm(r2, r2, 0, 31);
1725   }
1726 
1727   // Handle all the JSR stuff here, then exit.
1728   // It's much shorter and cleaner than intermingling with the non-JSR
1729   // normal-branch stuff occurring below.
1730 
1731   if (is_jsr) {
1732     // Pre-load the next target bytecode into rscratch1
1733     __ load_unsigned_byte(rscratch1, Address(rbcp, r2));
1734     // compute return address as bci
1735     __ ldr(rscratch2, Address(rmethod, Method::const_offset()));
1736     __ add(rscratch2, rscratch2,
1737            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1738     __ sub(r1, rbcp, rscratch2);
1739     __ push_i(r1);
1740     // Adjust the bcp by the 16-bit displacement in r2
1741     __ add(rbcp, rbcp, r2);
1742     __ dispatch_only(vtos);
1743     return;
1744   }
1745 
1746   // Normal (non-jsr) branch handling
1747 
1748   // Adjust the bcp by the displacement in r2
1749   __ add(rbcp, rbcp, r2);
1750 
1751   assert(UseLoopCounter || !UseOnStackReplacement,
1752          "on-stack-replacement requires loop counters");
1753   Label backedge_counter_overflow;
1754   Label profile_method;
1755   Label dispatch;
1756   if (UseLoopCounter) {
1757     // increment backedge counter for backward branches
1758     // r0: MDO
1759     // w1: MDO bumped taken-count
1760     // r2: target offset
1761     __ cmp(r2, zr);
1762     __ br(Assembler::GT, dispatch); // count only if backward branch
1763 
1764     // ECN: FIXME: This code smells
1765     // check if MethodCounters exists
1766     Label has_counters;
1767     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1768     __ cbnz(rscratch1, has_counters);
1769     __ push(r0);
1770     __ push(r1);
1771     __ push(r2);
1772     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1773             InterpreterRuntime::build_method_counters), rmethod);
1774     __ pop(r2);
1775     __ pop(r1);
1776     __ pop(r0);
1777     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1778     __ cbz(rscratch1, dispatch); // No MethodCounters allocated, OutOfMemory
1779     __ bind(has_counters);
1780 
1781     if (TieredCompilation) {
1782       Label no_mdo;
1783       int increment = InvocationCounter::count_increment;
1784       int mask = ((1 << Tier0BackedgeNotifyFreqLog) - 1) << InvocationCounter::count_shift;
1785       if (ProfileInterpreter) {
1786         // Are we profiling?
1787         __ ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
1788         __ cbz(r1, no_mdo);
1789         // Increment the MDO backedge counter
1790         const Address mdo_backedge_counter(r1, in_bytes(MethodData::backedge_counter_offset()) +
1791                                            in_bytes(InvocationCounter::counter_offset()));
1792         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1793                                    r0, rscratch2, false, Assembler::EQ, &backedge_counter_overflow);
1794         __ b(dispatch);
1795       }
1796       __ bind(no_mdo);
1797       // Increment backedge counter in MethodCounters*
1798       __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1799       __ increment_mask_and_jump(Address(rscratch1, be_offset), increment, mask,
1800                                  r0, rscratch2, false, Assembler::EQ, &backedge_counter_overflow);
1801     } else {
1802       // increment counter
1803       __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
1804       __ ldrw(r0, Address(rscratch2, be_offset));        // load backedge counter
1805       __ addw(rscratch1, r0, InvocationCounter::count_increment); // increment counter
1806       __ strw(rscratch1, Address(rscratch2, be_offset));        // store counter
1807 
1808       __ ldrw(r0, Address(rscratch2, inv_offset));    // load invocation counter
1809       __ andw(r0, r0, (unsigned)InvocationCounter::count_mask_value); // and the status bits
1810       __ addw(r0, r0, rscratch1);        // add both counters
1811 
1812       if (ProfileInterpreter) {
1813         // Test to see if we should create a method data oop
1814         __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterProfileLimit));
1815         __ ldrw(rscratch1, rscratch1);
1816         __ cmpw(r0, rscratch1);
1817         __ br(Assembler::LT, dispatch);
1818 
1819         // if no method data exists, go to profile method
1820         __ test_method_data_pointer(r0, profile_method);
1821 
1822         if (UseOnStackReplacement) {
1823           // check for overflow against w1 which is the MDO taken count
1824           __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
1825           __ ldrw(rscratch1, rscratch1);
1826           __ cmpw(r1, rscratch1);
1827           __ br(Assembler::LO, dispatch); // Intel == Assembler::below
1828 
1829           // When ProfileInterpreter is on, the backedge_count comes
1830           // from the MethodData*, which value does not get reset on
1831           // the call to frequency_counter_overflow().  To avoid
1832           // excessive calls to the overflow routine while the method is
1833           // being compiled, add a second test to make sure the overflow
1834           // function is called only once every overflow_frequency.
1835           const int overflow_frequency = 1024;
1836           __ andsw(r1, r1, overflow_frequency - 1);
1837           __ br(Assembler::EQ, backedge_counter_overflow);
1838 
1839         }
1840       } else {
1841         if (UseOnStackReplacement) {
1842           // check for overflow against w0, which is the sum of the
1843           // counters
1844           __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
1845           __ ldrw(rscratch1, rscratch1);
1846           __ cmpw(r0, rscratch1);
1847           __ br(Assembler::HS, backedge_counter_overflow); // Intel == Assembler::aboveEqual
1848         }
1849       }
1850     }
1851   }
1852   __ bind(dispatch);
1853 
1854   // Pre-load the next target bytecode into rscratch1
1855   __ load_unsigned_byte(rscratch1, Address(rbcp, 0));
1856 
1857   // continue with the bytecode @ target
1858   // rscratch1: target bytecode
1859   // rbcp: target bcp
1860   __ dispatch_only(vtos);
1861 
1862   if (UseLoopCounter) {
1863     if (ProfileInterpreter) {
1864       // Out-of-line code to allocate method data oop.
1865       __ bind(profile_method);
1866       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1867       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1868       __ set_method_data_pointer_for_bcp();
1869       __ b(dispatch);
1870     }
1871 
1872     if (TieredCompilation || UseOnStackReplacement) {
1873       // invocation counter overflow
1874       __ bind(backedge_counter_overflow);
1875       __ neg(r2, r2);
1876       __ add(r2, r2, rbcp);     // branch bcp
1877       // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
1878       __ call_VM(noreg,
1879                  CAST_FROM_FN_PTR(address,
1880                                   InterpreterRuntime::frequency_counter_overflow),
1881                  r2);
1882       if (!UseOnStackReplacement)
1883         __ b(dispatch);
1884     }
1885 
1886     if (UseOnStackReplacement) {
1887       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
1888 
1889       // r0: osr nmethod (osr ok) or NULL (osr not possible)
1890       // w1: target bytecode
1891       // r2: scratch
1892       __ cbz(r0, dispatch);     // test result -- no osr if null
1893       // nmethod may have been invalidated (VM may block upon call_VM return)
1894       __ ldrw(r2, Address(r0, nmethod::entry_bci_offset()));
1895       // InvalidOSREntryBci == -2 which overflows cmpw as unsigned
1896       // use cmnw against -InvalidOSREntryBci which does the same thing
1897       __ cmn(r2, -InvalidOSREntryBci);
1898       __ br(Assembler::EQ, dispatch);
1899 
1900       // We have the address of an on stack replacement routine in r0
1901       // We need to prepare to execute the OSR method. First we must
1902       // migrate the locals and monitors off of the stack.
1903 
1904       __ mov(r19, r0);                             // save the nmethod
1905 
1906       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
1907 
1908       // r0 is OSR buffer, move it to expected parameter location
1909       __ mov(j_rarg0, r0);
1910 
1911       // remove activation
1912       // get sender esp
1913       __ ldr(esp,
1914           Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
1915       // remove frame anchor
1916       __ leave();
1917       // Ensure compiled code always sees stack at proper alignment
1918       __ andr(sp, esp, -16);
1919 
1920       // and begin the OSR nmethod
1921       __ ldr(rscratch1, Address(r19, nmethod::osr_entry_point_offset()));
1922       __ br(rscratch1);
1923     }
1924   }
1925 }
1926 
1927 
1928 void TemplateTable::if_0cmp(Condition cc)
1929 {
1930   transition(itos, vtos);
1931   // assume branch is more often taken than not (loops use backward branches)
1932   Label not_taken;
1933   if (cc == equal)
1934     __ cbnzw(r0, not_taken);
1935   else if (cc == not_equal)
1936     __ cbzw(r0, not_taken);
1937   else {
1938     __ andsw(zr, r0, r0);
1939     __ br(j_not(cc), not_taken);
1940   }
1941 
1942   branch(false, false);
1943   __ bind(not_taken);
1944   __ profile_not_taken_branch(r0);
1945 }
1946 
1947 void TemplateTable::if_icmp(Condition cc)
1948 {
1949   transition(itos, vtos);
1950   // assume branch is more often taken than not (loops use backward branches)
1951   Label not_taken;
1952   __ pop_i(r1);
1953   __ cmpw(r1, r0, Assembler::LSL);
1954   __ br(j_not(cc), not_taken);
1955   branch(false, false);
1956   __ bind(not_taken);
1957   __ profile_not_taken_branch(r0);
1958 }
1959 
1960 void TemplateTable::if_nullcmp(Condition cc)
1961 {
1962   transition(atos, vtos);
1963   // assume branch is more often taken than not (loops use backward branches)
1964   Label not_taken;
1965   if (cc == equal)
1966     __ cbnz(r0, not_taken);
1967   else
1968     __ cbz(r0, not_taken);
1969   branch(false, false);
1970   __ bind(not_taken);
1971   __ profile_not_taken_branch(r0);
1972 }
1973 
1974 void TemplateTable::if_acmp(Condition cc)
1975 {
1976   transition(atos, vtos);
1977   // assume branch is more often taken than not (loops use backward branches)
1978   Label not_taken;
1979   __ pop_ptr(r1);
1980   __ cmp(r1, r0);
1981   __ br(j_not(cc), not_taken);
1982   branch(false, false);
1983   __ bind(not_taken);
1984   __ profile_not_taken_branch(r0);
1985 }
1986 
1987 void TemplateTable::ret() {
1988   transition(vtos, vtos);
1989   // We might be moving to a safepoint.  The thread which calls
1990   // Interpreter::notice_safepoints() will effectively flush its cache
1991   // when it makes a system call, but we need to do something to
1992   // ensure that we see the changed dispatch table.
1993   __ membar(MacroAssembler::LoadLoad);
1994 
1995   locals_index(r1);
1996   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
1997   __ profile_ret(r1, r2);
1998   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
1999   __ lea(rbcp, Address(rbcp, r1));
2000   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2001   __ dispatch_next(vtos);
2002 }
2003 
2004 void TemplateTable::wide_ret() {
2005   transition(vtos, vtos);
2006   locals_index_wide(r1);
2007   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2008   __ profile_ret(r1, r2);
2009   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2010   __ lea(rbcp, Address(rbcp, r1));
2011   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2012   __ dispatch_next(vtos);
2013 }
2014 
2015 
2016 void TemplateTable::tableswitch() {
2017   Label default_case, continue_execution;
2018   transition(itos, vtos);
2019   // align rbcp
2020   __ lea(r1, at_bcp(BytesPerInt));
2021   __ andr(r1, r1, -BytesPerInt);
2022   // load lo & hi
2023   __ ldrw(r2, Address(r1, BytesPerInt));
2024   __ ldrw(r3, Address(r1, 2 * BytesPerInt));
2025   __ rev32(r2, r2);
2026   __ rev32(r3, r3);
2027   // check against lo & hi
2028   __ cmpw(r0, r2);
2029   __ br(Assembler::LT, default_case);
2030   __ cmpw(r0, r3);
2031   __ br(Assembler::GT, default_case);
2032   // lookup dispatch offset
2033   __ subw(r0, r0, r2);
2034   __ lea(r3, Address(r1, r0, Address::uxtw(2)));
2035   __ ldrw(r3, Address(r3, 3 * BytesPerInt));
2036   __ profile_switch_case(r0, r1, r2);
2037   // continue execution
2038   __ bind(continue_execution);
2039   __ rev32(r3, r3);
2040   __ load_unsigned_byte(rscratch1, Address(rbcp, r3, Address::sxtw(0)));
2041   __ add(rbcp, rbcp, r3, ext::sxtw);
2042   __ dispatch_only(vtos);
2043   // handle default
2044   __ bind(default_case);
2045   __ profile_switch_default(r0);
2046   __ ldrw(r3, Address(r1, 0));
2047   __ b(continue_execution);
2048 }
2049 
2050 void TemplateTable::lookupswitch() {
2051   transition(itos, itos);
2052   __ stop("lookupswitch bytecode should have been rewritten");
2053 }
2054 
2055 void TemplateTable::fast_linearswitch() {
2056   transition(itos, vtos);
2057   Label loop_entry, loop, found, continue_execution;
2058   // bswap r0 so we can avoid bswapping the table entries
2059   __ rev32(r0, r0);
2060   // align rbcp
2061   __ lea(r19, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2062                                     // this instruction (change offsets
2063                                     // below)
2064   __ andr(r19, r19, -BytesPerInt);
2065   // set counter
2066   __ ldrw(r1, Address(r19, BytesPerInt));
2067   __ rev32(r1, r1);
2068   __ b(loop_entry);
2069   // table search
2070   __ bind(loop);
2071   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2072   __ ldrw(rscratch1, Address(rscratch1, 2 * BytesPerInt));
2073   __ cmpw(r0, rscratch1);
2074   __ br(Assembler::EQ, found);
2075   __ bind(loop_entry);
2076   __ subs(r1, r1, 1);
2077   __ br(Assembler::PL, loop);
2078   // default case
2079   __ profile_switch_default(r0);
2080   __ ldrw(r3, Address(r19, 0));
2081   __ b(continue_execution);
2082   // entry found -> get offset
2083   __ bind(found);
2084   __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2085   __ ldrw(r3, Address(rscratch1, 3 * BytesPerInt));
2086   __ profile_switch_case(r1, r0, r19);
2087   // continue execution
2088   __ bind(continue_execution);
2089   __ rev32(r3, r3);
2090   __ add(rbcp, rbcp, r3, ext::sxtw);
2091   __ ldrb(rscratch1, Address(rbcp, 0));
2092   __ dispatch_only(vtos);
2093 }
2094 
2095 void TemplateTable::fast_binaryswitch() {
2096   transition(itos, vtos);
2097   // Implementation using the following core algorithm:
2098   //
2099   // int binary_search(int key, LookupswitchPair* array, int n) {
2100   //   // Binary search according to "Methodik des Programmierens" by
2101   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2102   //   int i = 0;
2103   //   int j = n;
2104   //   while (i+1 < j) {
2105   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2106   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2107   //     // where a stands for the array and assuming that the (inexisting)
2108   //     // element a[n] is infinitely big.
2109   //     int h = (i + j) >> 1;
2110   //     // i < h < j
2111   //     if (key < array[h].fast_match()) {
2112   //       j = h;
2113   //     } else {
2114   //       i = h;
2115   //     }
2116   //   }
2117   //   // R: a[i] <= key < a[i+1] or Q
2118   //   // (i.e., if key is within array, i is the correct index)
2119   //   return i;
2120   // }
2121 
2122   // Register allocation
2123   const Register key   = r0; // already set (tosca)
2124   const Register array = r1;
2125   const Register i     = r2;
2126   const Register j     = r3;
2127   const Register h     = rscratch1;
2128   const Register temp  = rscratch2;
2129 
2130   // Find array start
2131   __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2132                                           // get rid of this
2133                                           // instruction (change
2134                                           // offsets below)
2135   __ andr(array, array, -BytesPerInt);
2136 
2137   // Initialize i & j
2138   __ mov(i, 0);                            // i = 0;
2139   __ ldrw(j, Address(array, -BytesPerInt)); // j = length(array);
2140 
2141   // Convert j into native byteordering
2142   __ rev32(j, j);
2143 
2144   // And start
2145   Label entry;
2146   __ b(entry);
2147 
2148   // binary search loop
2149   {
2150     Label loop;
2151     __ bind(loop);
2152     // int h = (i + j) >> 1;
2153     __ addw(h, i, j);                           // h = i + j;
2154     __ lsrw(h, h, 1);                                   // h = (i + j) >> 1;
2155     // if (key < array[h].fast_match()) {
2156     //   j = h;
2157     // } else {
2158     //   i = h;
2159     // }
2160     // Convert array[h].match to native byte-ordering before compare
2161     __ ldr(temp, Address(array, h, Address::lsl(3)));
2162     __ rev32(temp, temp);
2163     __ cmpw(key, temp);
2164     // j = h if (key <  array[h].fast_match())
2165     __ csel(j, h, j, Assembler::LT);
2166     // i = h if (key >= array[h].fast_match())
2167     __ csel(i, h, i, Assembler::GE);
2168     // while (i+1 < j)
2169     __ bind(entry);
2170     __ addw(h, i, 1);          // i+1
2171     __ cmpw(h, j);             // i+1 < j
2172     __ br(Assembler::LT, loop);
2173   }
2174 
2175   // end of binary search, result index is i (must check again!)
2176   Label default_case;
2177   // Convert array[i].match to native byte-ordering before compare
2178   __ ldr(temp, Address(array, i, Address::lsl(3)));
2179   __ rev32(temp, temp);
2180   __ cmpw(key, temp);
2181   __ br(Assembler::NE, default_case);
2182 
2183   // entry found -> j = offset
2184   __ add(j, array, i, ext::uxtx, 3);
2185   __ ldrw(j, Address(j, BytesPerInt));
2186   __ profile_switch_case(i, key, array);
2187   __ rev32(j, j);
2188   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2189   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2190   __ dispatch_only(vtos);
2191 
2192   // default case -> j = default offset
2193   __ bind(default_case);
2194   __ profile_switch_default(i);
2195   __ ldrw(j, Address(array, -2 * BytesPerInt));
2196   __ rev32(j, j);
2197   __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2198   __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2199   __ dispatch_only(vtos);
2200 }
2201 
2202 
2203 void TemplateTable::_return(TosState state)
2204 {
2205   transition(state, state);
2206   assert(_desc->calls_vm(),
2207          "inconsistent calls_vm information"); // call in remove_activation
2208 
2209   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2210     assert(state == vtos, "only valid state");
2211 
2212     __ ldr(c_rarg1, aaddress(0));
2213     __ load_klass(r3, c_rarg1);
2214     __ ldrw(r3, Address(r3, Klass::access_flags_offset()));
2215     __ tst(r3, JVM_ACC_HAS_FINALIZER);
2216     Label skip_register_finalizer;
2217     __ br(Assembler::EQ, skip_register_finalizer);
2218 
2219     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2220 
2221     __ bind(skip_register_finalizer);
2222   }
2223 
2224   // Issue a StoreStore barrier after all stores but before return
2225   // from any constructor for any class with a final field.  We don't
2226   // know if this is a finalizer, so we always do so.
2227   if (_desc->bytecode() == Bytecodes::_return)
2228     __ membar(MacroAssembler::StoreStore);
2229 
2230   // Narrow result if state is itos but result type is smaller.
2231   // Need to narrow in the return bytecode rather than in generate_return_entry
2232   // since compiled code callers expect the result to already be narrowed.
2233   if (state == itos) {
2234     __ narrow(r0);
2235   }
2236 
2237   __ remove_activation(state);
2238   __ ret(lr);
2239 }
2240 
2241 // ----------------------------------------------------------------------------
2242 // Volatile variables demand their effects be made known to all CPU's
2243 // in order.  Store buffers on most chips allow reads & writes to
2244 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2245 // without some kind of memory barrier (i.e., it's not sufficient that
2246 // the interpreter does not reorder volatile references, the hardware
2247 // also must not reorder them).
2248 //
2249 // According to the new Java Memory Model (JMM):
2250 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2251 //     writes act as aquire & release, so:
2252 // (2) A read cannot let unrelated NON-volatile memory refs that
2253 //     happen after the read float up to before the read.  It's OK for
2254 //     non-volatile memory refs that happen before the volatile read to
2255 //     float down below it.
2256 // (3) Similar a volatile write cannot let unrelated NON-volatile
2257 //     memory refs that happen BEFORE the write float down to after the
2258 //     write.  It's OK for non-volatile memory refs that happen after the
2259 //     volatile write to float up before it.
2260 //
2261 // We only put in barriers around volatile refs (they are expensive),
2262 // not _between_ memory refs (that would require us to track the
2263 // flavor of the previous memory refs).  Requirements (2) and (3)
2264 // require some barriers before volatile stores and after volatile
2265 // loads.  These nearly cover requirement (1) but miss the
2266 // volatile-store-volatile-load case.  This final case is placed after
2267 // volatile-stores although it could just as well go before
2268 // volatile-loads.
2269 
2270 void TemplateTable::resolve_cache_and_index(int byte_no,
2271                                             Register Rcache,
2272                                             Register index,
2273                                             size_t index_size) {
2274   const Register temp = r19;
2275   assert_different_registers(Rcache, index, temp);
2276 
2277   Label resolved;
2278   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2279   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2280   __ cmp(temp, (int) bytecode());  // have we resolved this bytecode?
2281   __ br(Assembler::EQ, resolved);
2282 
2283   // resolve first time through
2284   address entry;
2285   switch (bytecode()) {
2286   case Bytecodes::_getstatic:
2287   case Bytecodes::_putstatic:
2288   case Bytecodes::_getfield:
2289   case Bytecodes::_putfield:
2290     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_get_put);
2291     break;
2292   case Bytecodes::_invokevirtual:
2293   case Bytecodes::_invokespecial:
2294   case Bytecodes::_invokestatic:
2295   case Bytecodes::_invokeinterface:
2296     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invoke);
2297     break;
2298   case Bytecodes::_invokehandle:
2299     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invokehandle);
2300     break;
2301   case Bytecodes::_invokedynamic:
2302     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invokedynamic);
2303     break;
2304   default:
2305     fatal(err_msg("unexpected bytecode: %s", Bytecodes::name(bytecode())));
2306     break;
2307   }
2308   __ mov(temp, (int) bytecode());
2309   __ call_VM(noreg, entry, temp);
2310 
2311   // Update registers with resolved info
2312   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2313   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2314   // so all clients ofthis method must be modified accordingly
2315   __ bind(resolved);
2316 }
2317 
2318 // The Rcache and index registers must be set before call
2319 // n.b unlike x86 cache already includes the index offset
2320 void TemplateTable::load_field_cp_cache_entry(Register obj,
2321                                               Register cache,
2322                                               Register index,
2323                                               Register off,
2324                                               Register flags,
2325                                               bool is_static = false) {
2326   assert_different_registers(cache, index, flags, off);
2327 
2328   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2329   // Field offset
2330   __ ldr(off, Address(cache, in_bytes(cp_base_offset +
2331                                           ConstantPoolCacheEntry::f2_offset())));
2332   // Flags
2333   __ ldrw(flags, Address(cache, in_bytes(cp_base_offset +
2334                                            ConstantPoolCacheEntry::flags_offset())));
2335 
2336   // klass overwrite register
2337   if (is_static) {
2338     __ ldr(obj, Address(cache, in_bytes(cp_base_offset +
2339                                         ConstantPoolCacheEntry::f1_offset())));
2340     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2341     __ ldr(obj, Address(obj, mirror_offset));
2342   }
2343 }
2344 
2345 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2346                                                Register method,
2347                                                Register itable_index,
2348                                                Register flags,
2349                                                bool is_invokevirtual,
2350                                                bool is_invokevfinal, /*unused*/
2351                                                bool is_invokedynamic) {
2352   // setup registers
2353   const Register cache = rscratch2;
2354   const Register index = r4;
2355   assert_different_registers(method, flags);
2356   assert_different_registers(method, cache, index);
2357   assert_different_registers(itable_index, flags);
2358   assert_different_registers(itable_index, cache, index);
2359   // determine constant pool cache field offsets
2360   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2361   const int method_offset = in_bytes(
2362     ConstantPoolCache::base_offset() +
2363       (is_invokevirtual
2364        ? ConstantPoolCacheEntry::f2_offset()
2365        : ConstantPoolCacheEntry::f1_offset()));
2366   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2367                                     ConstantPoolCacheEntry::flags_offset());
2368   // access constant pool cache fields
2369   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2370                                     ConstantPoolCacheEntry::f2_offset());
2371 
2372   size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2373   resolve_cache_and_index(byte_no, cache, index, index_size);
2374   __ ldr(method, Address(cache, method_offset));
2375 
2376   if (itable_index != noreg) {
2377     __ ldr(itable_index, Address(cache, index_offset));
2378   }
2379   __ ldrw(flags, Address(cache, flags_offset));
2380 }
2381 
2382 
2383 // The registers cache and index expected to be set before call.
2384 // Correct values of the cache and index registers are preserved.
2385 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2386                                             bool is_static, bool has_tos) {
2387   // do the JVMTI work here to avoid disturbing the register state below
2388   // We use c_rarg registers here because we want to use the register used in
2389   // the call to the VM
2390   if (JvmtiExport::can_post_field_access()) {
2391     // Check to see if a field access watch has been set before we
2392     // take the time to call into the VM.
2393     Label L1;
2394     assert_different_registers(cache, index, r0);
2395     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2396     __ ldrw(r0, Address(rscratch1));
2397     __ cbzw(r0, L1);
2398 
2399     __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2400     __ lea(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2401 
2402     if (is_static) {
2403       __ mov(c_rarg1, zr); // NULL object reference
2404     } else {
2405       __ ldr(c_rarg1, at_tos()); // get object pointer without popping it
2406       __ verify_oop(c_rarg1);
2407     }
2408     // c_rarg1: object pointer or NULL
2409     // c_rarg2: cache entry pointer
2410     // c_rarg3: jvalue object on the stack
2411     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2412                                        InterpreterRuntime::post_field_access),
2413                c_rarg1, c_rarg2, c_rarg3);
2414     __ get_cache_and_index_at_bcp(cache, index, 1);
2415     __ bind(L1);
2416   }
2417 }
2418 
2419 void TemplateTable::pop_and_check_object(Register r)
2420 {
2421   __ pop_ptr(r);
2422   __ null_check(r);  // for field access must check obj.
2423   __ verify_oop(r);
2424 }
2425 
2426 void TemplateTable::getfield_or_static(int byte_no, bool is_static)
2427 {
2428   const Register cache = r2;
2429   const Register index = r3;
2430   const Register obj   = r4;
2431   const Register off   = r19;
2432   const Register flags = r0;
2433   const Register raw_flags = r6;
2434   const Register bc    = r4; // uses same reg as obj, so don't mix them
2435 
2436   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2437   jvmti_post_field_access(cache, index, is_static, false);
2438   load_field_cp_cache_entry(obj, cache, index, off, raw_flags, is_static);
2439 
2440   if (!is_static) {
2441     // obj is on the stack
2442     pop_and_check_object(obj);
2443   }
2444 
2445   // 8179954: We need to make sure that the code generated for
2446   // volatile accesses forms a sequentially-consistent set of
2447   // operations when combined with STLR and LDAR.  Without a leading
2448   // membar it's possible for a simple Dekker test to fail if loads
2449   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
2450   // the stores in one method and we interpret the loads in another.
2451   if (! UseBarriersForVolatile) {
2452     Label notVolatile;
2453     __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2454     __ membar(MacroAssembler::AnyAny);
2455     __ bind(notVolatile);
2456   }
2457 
2458   const Address field(obj, off);
2459 
2460   Label Done, notByte, notBool, notInt, notShort, notChar,
2461               notLong, notFloat, notObj, notDouble;
2462 
2463   // x86 uses a shift and mask or wings it with a shift plus assert
2464   // the mask is not needed. aarch64 just uses bitfield extract
2465   __ ubfxw(flags, raw_flags, ConstantPoolCacheEntry::tos_state_shift,
2466            ConstantPoolCacheEntry::tos_state_bits);
2467 
2468   assert(btos == 0, "change code, btos != 0");
2469   __ cbnz(flags, notByte);
2470 
2471   // btos
2472   __ load_signed_byte(r0, field);
2473   __ push(btos);
2474   // Rewrite bytecode to be faster
2475   if (!is_static) {
2476     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2477   }
2478   __ b(Done);
2479 
2480   __ bind(notByte);
2481   __ cmp(flags, ztos);
2482   __ br(Assembler::NE, notBool);
2483 
2484   // ztos (same code as btos)
2485   __ ldrsb(r0, field);
2486   __ push(ztos);
2487   // Rewrite bytecode to be faster
2488   if (!is_static) {
2489     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2490     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2491   }
2492   __ b(Done);
2493 
2494   __ bind(notBool);
2495   __ cmp(flags, atos);
2496   __ br(Assembler::NE, notObj);
2497   // atos
2498   __ load_heap_oop(r0, field);
2499   __ push(atos);
2500   if (!is_static) {
2501     patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2502   }
2503   __ b(Done);
2504 
2505   __ bind(notObj);
2506   __ cmp(flags, itos);
2507   __ br(Assembler::NE, notInt);
2508   // itos
2509   __ ldrw(r0, field);
2510   __ push(itos);
2511   // Rewrite bytecode to be faster
2512   if (!is_static) {
2513     patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2514   }
2515   __ b(Done);
2516 
2517   __ bind(notInt);
2518   __ cmp(flags, ctos);
2519   __ br(Assembler::NE, notChar);
2520   // ctos
2521   __ load_unsigned_short(r0, field);
2522   __ push(ctos);
2523   // Rewrite bytecode to be faster
2524   if (!is_static) {
2525     patch_bytecode(Bytecodes::_fast_cgetfield, bc, r1);
2526   }
2527   __ b(Done);
2528 
2529   __ bind(notChar);
2530   __ cmp(flags, stos);
2531   __ br(Assembler::NE, notShort);
2532   // stos
2533   __ load_signed_short(r0, field);
2534   __ push(stos);
2535   // Rewrite bytecode to be faster
2536   if (!is_static) {
2537     patch_bytecode(Bytecodes::_fast_sgetfield, bc, r1);
2538   }
2539   __ b(Done);
2540 
2541   __ bind(notShort);
2542   __ cmp(flags, ltos);
2543   __ br(Assembler::NE, notLong);
2544   // ltos
2545   __ ldr(r0, field);
2546   __ push(ltos);
2547   // Rewrite bytecode to be faster
2548   if (!is_static) {
2549     patch_bytecode(Bytecodes::_fast_lgetfield, bc, r1);
2550   }
2551   __ b(Done);
2552 
2553   __ bind(notLong);
2554   __ cmp(flags, ftos);
2555   __ br(Assembler::NE, notFloat);
2556   // ftos
2557   __ ldrs(v0, field);
2558   __ push(ftos);
2559   // Rewrite bytecode to be faster
2560   if (!is_static) {
2561     patch_bytecode(Bytecodes::_fast_fgetfield, bc, r1);
2562   }
2563   __ b(Done);
2564 
2565   __ bind(notFloat);
2566 #ifdef ASSERT
2567   __ cmp(flags, dtos);
2568   __ br(Assembler::NE, notDouble);
2569 #endif
2570   // dtos
2571   __ ldrd(v0, field);
2572   __ push(dtos);
2573   // Rewrite bytecode to be faster
2574   if (!is_static) {
2575     patch_bytecode(Bytecodes::_fast_dgetfield, bc, r1);
2576   }
2577 #ifdef ASSERT
2578   __ b(Done);
2579 
2580   __ bind(notDouble);
2581   __ stop("Bad state");
2582 #endif
2583 
2584   __ bind(Done);
2585 
2586   Label notVolatile;
2587   __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2588   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2589   __ bind(notVolatile);
2590 }
2591 
2592 
2593 void TemplateTable::getfield(int byte_no)
2594 {
2595   getfield_or_static(byte_no, false);
2596 }
2597 
2598 void TemplateTable::getstatic(int byte_no)
2599 {
2600   getfield_or_static(byte_no, true);
2601 }
2602 
2603 // The registers cache and index expected to be set before call.
2604 // The function may destroy various registers, just not the cache and index registers.
2605 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2606   transition(vtos, vtos);
2607 
2608   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2609 
2610   if (JvmtiExport::can_post_field_modification()) {
2611     // Check to see if a field modification watch has been set before
2612     // we take the time to call into the VM.
2613     Label L1;
2614     assert_different_registers(cache, index, r0);
2615     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2616     __ ldrw(r0, Address(rscratch1));
2617     __ cbz(r0, L1);
2618 
2619     __ get_cache_and_index_at_bcp(c_rarg2, rscratch1, 1);
2620 
2621     if (is_static) {
2622       // Life is simple.  Null out the object pointer.
2623       __ mov(c_rarg1, zr);
2624     } else {
2625       // Life is harder. The stack holds the value on top, followed by
2626       // the object.  We don't know the size of the value, though; it
2627       // could be one or two words depending on its type. As a result,
2628       // we must find the type to determine where the object is.
2629       __ ldrw(c_rarg3, Address(c_rarg2,
2630                                in_bytes(cp_base_offset +
2631                                         ConstantPoolCacheEntry::flags_offset())));
2632       __ lsr(c_rarg3, c_rarg3,
2633              ConstantPoolCacheEntry::tos_state_shift);
2634       ConstantPoolCacheEntry::verify_tos_state_shift();
2635       Label nope2, done, ok;
2636       __ ldr(c_rarg1, at_tos_p1());  // initially assume a one word jvalue
2637       __ cmpw(c_rarg3, ltos);
2638       __ br(Assembler::EQ, ok);
2639       __ cmpw(c_rarg3, dtos);
2640       __ br(Assembler::NE, nope2);
2641       __ bind(ok);
2642       __ ldr(c_rarg1, at_tos_p2()); // ltos (two word jvalue)
2643       __ bind(nope2);
2644     }
2645     // cache entry pointer
2646     __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2647     // object (tos)
2648     __ mov(c_rarg3, esp);
2649     // c_rarg1: object pointer set up above (NULL if static)
2650     // c_rarg2: cache entry pointer
2651     // c_rarg3: jvalue object on the stack
2652     __ call_VM(noreg,
2653                CAST_FROM_FN_PTR(address,
2654                                 InterpreterRuntime::post_field_modification),
2655                c_rarg1, c_rarg2, c_rarg3);
2656     __ get_cache_and_index_at_bcp(cache, index, 1);
2657     __ bind(L1);
2658   }
2659 }
2660 
2661 void TemplateTable::putfield_or_static(int byte_no, bool is_static) {
2662   transition(vtos, vtos);
2663 
2664   const Register cache = r2;
2665   const Register index = r3;
2666   const Register obj   = r2;
2667   const Register off   = r19;
2668   const Register flags = r0;
2669   const Register bc    = r4;
2670 
2671   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2672   jvmti_post_field_mod(cache, index, is_static);
2673   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2674 
2675   Label Done;
2676   __ mov(r5, flags);
2677 
2678   {
2679     Label notVolatile;
2680     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2681     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2682     __ bind(notVolatile);
2683   }
2684 
2685   // field address
2686   const Address field(obj, off);
2687 
2688   Label notByte, notBool, notInt, notShort, notChar,
2689         notLong, notFloat, notObj, notDouble;
2690 
2691   // x86 uses a shift and mask or wings it with a shift plus assert
2692   // the mask is not needed. aarch64 just uses bitfield extract
2693   __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
2694 
2695   assert(btos == 0, "change code, btos != 0");
2696   __ cbnz(flags, notByte);
2697 
2698   // btos
2699   {
2700     __ pop(btos);
2701     if (!is_static) pop_and_check_object(obj);
2702     __ strb(r0, field);
2703     if (!is_static) {
2704       patch_bytecode(Bytecodes::_fast_bputfield, bc, r1, true, byte_no);
2705     }
2706     __ b(Done);
2707   }
2708 
2709   __ bind(notByte);
2710   __ cmp(flags, ztos);
2711   __ br(Assembler::NE, notBool);
2712 
2713   // ztos
2714   {
2715     __ pop(ztos);
2716     if (!is_static) pop_and_check_object(obj);
2717     __ andw(r0, r0, 0x1);
2718     __ strb(r0, field);
2719     if (!is_static) {
2720       patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
2721     }
2722     __ b(Done);
2723   }
2724 
2725   __ bind(notBool);
2726   __ cmp(flags, atos);
2727   __ br(Assembler::NE, notObj);
2728 
2729   // atos
2730   {
2731     __ pop(atos);
2732     if (!is_static) pop_and_check_object(obj);
2733     // Store into the field
2734     do_oop_store(_masm, field, r0, _bs->kind(), false);
2735     if (!is_static) {
2736       patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
2737     }
2738     __ b(Done);
2739   }
2740 
2741   __ bind(notObj);
2742   __ cmp(flags, itos);
2743   __ br(Assembler::NE, notInt);
2744 
2745   // itos
2746   {
2747     __ pop(itos);
2748     if (!is_static) pop_and_check_object(obj);
2749     __ strw(r0, field);
2750     if (!is_static) {
2751       patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
2752     }
2753     __ b(Done);
2754   }
2755 
2756   __ bind(notInt);
2757   __ cmp(flags, ctos);
2758   __ br(Assembler::NE, notChar);
2759 
2760   // ctos
2761   {
2762     __ pop(ctos);
2763     if (!is_static) pop_and_check_object(obj);
2764     __ strh(r0, field);
2765     if (!is_static) {
2766       patch_bytecode(Bytecodes::_fast_cputfield, bc, r1, true, byte_no);
2767     }
2768     __ b(Done);
2769   }
2770 
2771   __ bind(notChar);
2772   __ cmp(flags, stos);
2773   __ br(Assembler::NE, notShort);
2774 
2775   // stos
2776   {
2777     __ pop(stos);
2778     if (!is_static) pop_and_check_object(obj);
2779     __ strh(r0, field);
2780     if (!is_static) {
2781       patch_bytecode(Bytecodes::_fast_sputfield, bc, r1, true, byte_no);
2782     }
2783     __ b(Done);
2784   }
2785 
2786   __ bind(notShort);
2787   __ cmp(flags, ltos);
2788   __ br(Assembler::NE, notLong);
2789 
2790   // ltos
2791   {
2792     __ pop(ltos);
2793     if (!is_static) pop_and_check_object(obj);
2794     __ str(r0, field);
2795     if (!is_static) {
2796       patch_bytecode(Bytecodes::_fast_lputfield, bc, r1, true, byte_no);
2797     }
2798     __ b(Done);
2799   }
2800 
2801   __ bind(notLong);
2802   __ cmp(flags, ftos);
2803   __ br(Assembler::NE, notFloat);
2804 
2805   // ftos
2806   {
2807     __ pop(ftos);
2808     if (!is_static) pop_and_check_object(obj);
2809     __ strs(v0, field);
2810     if (!is_static) {
2811       patch_bytecode(Bytecodes::_fast_fputfield, bc, r1, true, byte_no);
2812     }
2813     __ b(Done);
2814   }
2815 
2816   __ bind(notFloat);
2817 #ifdef ASSERT
2818   __ cmp(flags, dtos);
2819   __ br(Assembler::NE, notDouble);
2820 #endif
2821 
2822   // dtos
2823   {
2824     __ pop(dtos);
2825     if (!is_static) pop_and_check_object(obj);
2826     __ strd(v0, field);
2827     if (!is_static) {
2828       patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
2829     }
2830   }
2831 
2832 #ifdef ASSERT
2833   __ b(Done);
2834 
2835   __ bind(notDouble);
2836   __ stop("Bad state");
2837 #endif
2838 
2839   __ bind(Done);
2840 
2841   {
2842     Label notVolatile;
2843     __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2844     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
2845     __ bind(notVolatile);
2846   }
2847 }
2848 
2849 void TemplateTable::putfield(int byte_no)
2850 {
2851   putfield_or_static(byte_no, false);
2852 }
2853 
2854 void TemplateTable::putstatic(int byte_no) {
2855   putfield_or_static(byte_no, true);
2856 }
2857 
2858 void TemplateTable::jvmti_post_fast_field_mod()
2859 {
2860   if (JvmtiExport::can_post_field_modification()) {
2861     // Check to see if a field modification watch has been set before
2862     // we take the time to call into the VM.
2863     Label L2;
2864     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2865     __ ldrw(c_rarg3, Address(rscratch1));
2866     __ cbzw(c_rarg3, L2);
2867     __ pop_ptr(r19);                  // copy the object pointer from tos
2868     __ verify_oop(r19);
2869     __ push_ptr(r19);                 // put the object pointer back on tos
2870     // Save tos values before call_VM() clobbers them. Since we have
2871     // to do it for every data type, we use the saved values as the
2872     // jvalue object.
2873     switch (bytecode()) {          // load values into the jvalue object
2874     case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
2875     case Bytecodes::_fast_bputfield: // fall through
2876     case Bytecodes::_fast_zputfield: // fall through
2877     case Bytecodes::_fast_sputfield: // fall through
2878     case Bytecodes::_fast_cputfield: // fall through
2879     case Bytecodes::_fast_iputfield: __ push_i(r0); break;
2880     case Bytecodes::_fast_dputfield: __ push_d(); break;
2881     case Bytecodes::_fast_fputfield: __ push_f(); break;
2882     case Bytecodes::_fast_lputfield: __ push_l(r0); break;
2883 
2884     default:
2885       ShouldNotReachHere();
2886     }
2887     __ mov(c_rarg3, esp);             // points to jvalue on the stack
2888     // access constant pool cache entry
2889     __ get_cache_entry_pointer_at_bcp(c_rarg2, r0, 1);
2890     __ verify_oop(r19);
2891     // r19: object pointer copied above
2892     // c_rarg2: cache entry pointer
2893     // c_rarg3: jvalue object on the stack
2894     __ call_VM(noreg,
2895                CAST_FROM_FN_PTR(address,
2896                                 InterpreterRuntime::post_field_modification),
2897                r19, c_rarg2, c_rarg3);
2898 
2899     switch (bytecode()) {             // restore tos values
2900     case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
2901     case Bytecodes::_fast_bputfield: // fall through
2902     case Bytecodes::_fast_zputfield: // fall through
2903     case Bytecodes::_fast_sputfield: // fall through
2904     case Bytecodes::_fast_cputfield: // fall through
2905     case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
2906     case Bytecodes::_fast_dputfield: __ pop_d(); break;
2907     case Bytecodes::_fast_fputfield: __ pop_f(); break;
2908     case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
2909     }
2910     __ bind(L2);
2911   }
2912 }
2913 
2914 void TemplateTable::fast_storefield(TosState state)
2915 {
2916   transition(state, vtos);
2917 
2918   ByteSize base = ConstantPoolCache::base_offset();
2919 
2920   jvmti_post_fast_field_mod();
2921 
2922   // access constant pool cache
2923   __ get_cache_and_index_at_bcp(r2, r1, 1);
2924 
2925   // Must prevent reordering of the following cp cache loads with bytecode load
2926   __ membar(MacroAssembler::LoadLoad);
2927 
2928   // test for volatile with r3
2929   __ ldrw(r3, Address(r2, in_bytes(base +
2930                                    ConstantPoolCacheEntry::flags_offset())));
2931 
2932   // replace index with field offset from cache entry
2933   __ ldr(r1, Address(r2, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
2934 
2935   {
2936     Label notVolatile;
2937     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2938     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2939     __ bind(notVolatile);
2940   }
2941 
2942   Label notVolatile;
2943 
2944   // Get object from stack
2945   pop_and_check_object(r2);
2946 
2947   // field address
2948   const Address field(r2, r1);
2949 
2950   // access field
2951   switch (bytecode()) {
2952   case Bytecodes::_fast_aputfield:
2953     do_oop_store(_masm, field, r0, _bs->kind(), false);
2954     break;
2955   case Bytecodes::_fast_lputfield:
2956     __ str(r0, field);
2957     break;
2958   case Bytecodes::_fast_iputfield:
2959     __ strw(r0, field);
2960     break;
2961   case Bytecodes::_fast_zputfield:
2962     __ andw(r0, r0, 0x1);  // boolean is true if LSB is 1
2963     // fall through to bputfield
2964   case Bytecodes::_fast_bputfield:
2965     __ strb(r0, field);
2966     break;
2967   case Bytecodes::_fast_sputfield:
2968     // fall through
2969   case Bytecodes::_fast_cputfield:
2970     __ strh(r0, field);
2971     break;
2972   case Bytecodes::_fast_fputfield:
2973     __ strs(v0, field);
2974     break;
2975   case Bytecodes::_fast_dputfield:
2976     __ strd(v0, field);
2977     break;
2978   default:
2979     ShouldNotReachHere();
2980   }
2981 
2982   {
2983     Label notVolatile;
2984     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2985     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
2986     __ bind(notVolatile);
2987   }
2988 }
2989 
2990 
2991 void TemplateTable::fast_accessfield(TosState state)
2992 {
2993   transition(atos, state);
2994   // Do the JVMTI work here to avoid disturbing the register state below
2995   if (JvmtiExport::can_post_field_access()) {
2996     // Check to see if a field access watch has been set before we
2997     // take the time to call into the VM.
2998     Label L1;
2999     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
3000     __ ldrw(r2, Address(rscratch1));
3001     __ cbzw(r2, L1);
3002     // access constant pool cache entry
3003     __ get_cache_entry_pointer_at_bcp(c_rarg2, rscratch2, 1);
3004     __ verify_oop(r0);
3005     __ push_ptr(r0);  // save object pointer before call_VM() clobbers it
3006     __ mov(c_rarg1, r0);
3007     // c_rarg1: object pointer copied above
3008     // c_rarg2: cache entry pointer
3009     __ call_VM(noreg,
3010                CAST_FROM_FN_PTR(address,
3011                                 InterpreterRuntime::post_field_access),
3012                c_rarg1, c_rarg2);
3013     __ pop_ptr(r0); // restore object pointer
3014     __ bind(L1);
3015   }
3016 
3017   // access constant pool cache
3018   __ get_cache_and_index_at_bcp(r2, r1, 1);
3019 
3020   // Must prevent reordering of the following cp cache loads with bytecode load
3021   __ membar(MacroAssembler::LoadLoad);
3022 
3023   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3024                                   ConstantPoolCacheEntry::f2_offset())));
3025   __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3026                                    ConstantPoolCacheEntry::flags_offset())));
3027 
3028   // r0: object
3029   __ verify_oop(r0);
3030   __ null_check(r0);
3031   const Address field(r0, r1);
3032 
3033   // 8179954: We need to make sure that the code generated for
3034   // volatile accesses forms a sequentially-consistent set of
3035   // operations when combined with STLR and LDAR.  Without a leading
3036   // membar it's possible for a simple Dekker test to fail if loads
3037   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3038   // the stores in one method and we interpret the loads in another.
3039   if (! UseBarriersForVolatile) {
3040     Label notVolatile;
3041     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3042     __ membar(MacroAssembler::AnyAny);
3043     __ bind(notVolatile);
3044   }
3045 
3046   // access field
3047   switch (bytecode()) {
3048   case Bytecodes::_fast_agetfield:
3049     __ load_heap_oop(r0, field);
3050     __ verify_oop(r0);
3051     break;
3052   case Bytecodes::_fast_lgetfield:
3053     __ ldr(r0, field);
3054     break;
3055   case Bytecodes::_fast_igetfield:
3056     __ ldrw(r0, field);
3057     break;
3058   case Bytecodes::_fast_bgetfield:
3059     __ load_signed_byte(r0, field);
3060     break;
3061   case Bytecodes::_fast_sgetfield:
3062     __ load_signed_short(r0, field);
3063     break;
3064   case Bytecodes::_fast_cgetfield:
3065     __ load_unsigned_short(r0, field);
3066     break;
3067   case Bytecodes::_fast_fgetfield:
3068     __ ldrs(v0, field);
3069     break;
3070   case Bytecodes::_fast_dgetfield:
3071     __ ldrd(v0, field);
3072     break;
3073   default:
3074     ShouldNotReachHere();
3075   }
3076   {
3077     Label notVolatile;
3078     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3079     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3080     __ bind(notVolatile);
3081   }
3082 }
3083 
3084 void TemplateTable::fast_xaccess(TosState state)
3085 {
3086   transition(vtos, state);
3087 
3088   // get receiver
3089   __ ldr(r0, aaddress(0));
3090   // access constant pool cache
3091   __ get_cache_and_index_at_bcp(r2, r3, 2);
3092   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3093                                   ConstantPoolCacheEntry::f2_offset())));
3094 
3095   // 8179954: We need to make sure that the code generated for
3096   // volatile accesses forms a sequentially-consistent set of
3097   // operations when combined with STLR and LDAR.  Without a leading
3098   // membar it's possible for a simple Dekker test to fail if loads
3099   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3100   // the stores in one method and we interpret the loads in another.
3101   if (! UseBarriersForVolatile) {
3102     Label notVolatile;
3103     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3104                                      ConstantPoolCacheEntry::flags_offset())));
3105     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3106     __ membar(MacroAssembler::AnyAny);
3107     __ bind(notVolatile);
3108   }
3109 
3110   // make sure exception is reported in correct bcp range (getfield is
3111   // next instruction)
3112   __ increment(rbcp);
3113   __ null_check(r0);
3114   switch (state) {
3115   case itos:
3116     __ ldrw(r0, Address(r0, r1, Address::lsl(0)));
3117     break;
3118   case atos:
3119     __ load_heap_oop(r0, Address(r0, r1, Address::lsl(0)));
3120     __ verify_oop(r0);
3121     break;
3122   case ftos:
3123     __ ldrs(v0, Address(r0, r1, Address::lsl(0)));
3124     break;
3125   default:
3126     ShouldNotReachHere();
3127   }
3128 
3129   {
3130     Label notVolatile;
3131     __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3132                                      ConstantPoolCacheEntry::flags_offset())));
3133     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3134     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3135     __ bind(notVolatile);
3136   }
3137 
3138   __ decrement(rbcp);
3139 }
3140 
3141 
3142 
3143 //-----------------------------------------------------------------------------
3144 // Calls
3145 
3146 void TemplateTable::count_calls(Register method, Register temp)
3147 {
3148   __ call_Unimplemented();
3149 }
3150 
3151 void TemplateTable::prepare_invoke(int byte_no,
3152                                    Register method, // linked method (or i-klass)
3153                                    Register index,  // itable index, MethodType, etc.
3154                                    Register recv,   // if caller wants to see it
3155                                    Register flags   // if caller wants to test it
3156                                    ) {
3157   // determine flags
3158   Bytecodes::Code code = bytecode();
3159   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3160   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3161   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3162   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3163   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3164   const bool load_receiver       = (recv  != noreg);
3165   const bool save_flags          = (flags != noreg);
3166   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3167   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3168   assert(flags == noreg || flags == r3, "");
3169   assert(recv  == noreg || recv  == r2, "");
3170 
3171   // setup registers & access constant pool cache
3172   if (recv  == noreg)  recv  = r2;
3173   if (flags == noreg)  flags = r3;
3174   assert_different_registers(method, index, recv, flags);
3175 
3176   // save 'interpreter return address'
3177   __ save_bcp();
3178 
3179   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3180 
3181   // maybe push appendix to arguments (just before return address)
3182   if (is_invokedynamic || is_invokehandle) {
3183     Label L_no_push;
3184     __ tbz(flags, ConstantPoolCacheEntry::has_appendix_shift, L_no_push);
3185     // Push the appendix as a trailing parameter.
3186     // This must be done before we get the receiver,
3187     // since the parameter_size includes it.
3188     __ push(r19);
3189     __ mov(r19, index);
3190     assert(ConstantPoolCacheEntry::_indy_resolved_references_appendix_offset == 0, "appendix expected at index+0");
3191     __ load_resolved_reference_at_index(index, r19);
3192     __ pop(r19);
3193     __ push(index);  // push appendix (MethodType, CallSite, etc.)
3194     __ bind(L_no_push);
3195   }
3196 
3197   // load receiver if needed (note: no return address pushed yet)
3198   if (load_receiver) {
3199     __ andw(recv, flags, ConstantPoolCacheEntry::parameter_size_mask);
3200     // FIXME -- is this actually correct? looks like it should be 2
3201     // const int no_return_pc_pushed_yet = -1;  // argument slot correction before we push return address
3202     // const int receiver_is_at_end      = -1;  // back off one slot to get receiver
3203     // Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
3204     // __ movptr(recv, recv_addr);
3205     __ add(rscratch1, esp, recv, ext::uxtx, 3); // FIXME: uxtb here?
3206     __ ldr(recv, Address(rscratch1, -Interpreter::expr_offset_in_bytes(1)));
3207     __ verify_oop(recv);
3208   }
3209 
3210   // compute return type
3211   // x86 uses a shift and mask or wings it with a shift plus assert
3212   // the mask is not needed. aarch64 just uses bitfield extract
3213   __ ubfxw(rscratch2, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
3214   // load return address
3215   {
3216     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3217     __ mov(rscratch1, table_addr);
3218     __ ldr(lr, Address(rscratch1, rscratch2, Address::lsl(3)));
3219   }
3220 }
3221 
3222 
3223 void TemplateTable::invokevirtual_helper(Register index,
3224                                          Register recv,
3225                                          Register flags)
3226 {
3227   // Uses temporary registers r0, r3
3228   assert_different_registers(index, recv, r0, r3);
3229   // Test for an invoke of a final method
3230   Label notFinal;
3231   __ tbz(flags, ConstantPoolCacheEntry::is_vfinal_shift, notFinal);
3232 
3233   const Register method = index;  // method must be rmethod
3234   assert(method == rmethod,
3235          "methodOop must be rmethod for interpreter calling convention");
3236 
3237   // do the call - the index is actually the method to call
3238   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3239 
3240   // It's final, need a null check here!
3241   __ null_check(recv);
3242 
3243   // profile this call
3244   __ profile_final_call(r0);
3245   __ profile_arguments_type(r0, method, r4, true);
3246 
3247   __ jump_from_interpreted(method, r0);
3248 
3249   __ bind(notFinal);
3250 
3251   // get receiver klass
3252   __ null_check(recv, oopDesc::klass_offset_in_bytes());
3253   __ load_klass(r0, recv);
3254 
3255   // profile this call
3256   __ profile_virtual_call(r0, rlocals, r3);
3257 
3258   // get target methodOop & entry point
3259   __ lookup_virtual_method(r0, index, method);
3260   __ profile_arguments_type(r3, method, r4, true);
3261   // FIXME -- this looks completely redundant. is it?
3262   // __ ldr(r3, Address(method, Method::interpreter_entry_offset()));
3263   __ jump_from_interpreted(method, r3);
3264 }
3265 
3266 void TemplateTable::invokevirtual(int byte_no)
3267 {
3268   transition(vtos, vtos);
3269   assert(byte_no == f2_byte, "use this argument");
3270 
3271   prepare_invoke(byte_no, rmethod, noreg, r2, r3);
3272 
3273   // rmethod: index (actually a Method*)
3274   // r2: receiver
3275   // r3: flags
3276 
3277   invokevirtual_helper(rmethod, r2, r3);
3278 }
3279 
3280 void TemplateTable::invokespecial(int byte_no)
3281 {
3282   transition(vtos, vtos);
3283   assert(byte_no == f1_byte, "use this argument");
3284 
3285   prepare_invoke(byte_no, rmethod, noreg,  // get f1 Method*
3286                  r2);  // get receiver also for null check
3287   __ verify_oop(r2);
3288   __ null_check(r2);
3289   // do the call
3290   __ profile_call(r0);
3291   __ profile_arguments_type(r0, rmethod, rbcp, false);
3292   __ jump_from_interpreted(rmethod, r0);
3293 }
3294 
3295 void TemplateTable::invokestatic(int byte_no)
3296 {
3297   transition(vtos, vtos);
3298   assert(byte_no == f1_byte, "use this argument");
3299 
3300   prepare_invoke(byte_no, rmethod);  // get f1 Method*
3301   // do the call
3302   __ profile_call(r0);
3303   __ profile_arguments_type(r0, rmethod, r4, false);
3304   __ jump_from_interpreted(rmethod, r0);
3305 }
3306 
3307 void TemplateTable::fast_invokevfinal(int byte_no)
3308 {
3309   __ call_Unimplemented();
3310 }
3311 
3312 void TemplateTable::invokeinterface(int byte_no) {
3313   transition(vtos, vtos);
3314   assert(byte_no == f1_byte, "use this argument");
3315 
3316   prepare_invoke(byte_no, r0, rmethod,  // get f1 Klass*, f2 Method*
3317                  r2, r3); // recv, flags
3318 
3319   // r0: interface klass (from f1)
3320   // rmethod: method (from f2)
3321   // r2: receiver
3322   // r3: flags
3323 
3324   // Special case of invokeinterface called for virtual method of
3325   // java.lang.Object.  See cpCacheOop.cpp for details.
3326   // This code isn't produced by javac, but could be produced by
3327   // another compliant java compiler.
3328   Label notMethod;
3329   __ tbz(r3, ConstantPoolCacheEntry::is_forced_virtual_shift, notMethod);
3330 
3331   invokevirtual_helper(rmethod, r2, r3);
3332   __ bind(notMethod);
3333 
3334   // Get receiver klass into r3 - also a null check
3335   __ restore_locals();
3336   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3337   __ load_klass(r3, r2);
3338 
3339   Label no_such_interface, no_such_method;
3340 
3341   // Receiver subtype check against REFC.
3342   // Superklass in r0. Subklass in r3. Blows rscratch2, r13.
3343   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3344                              r3, r0, noreg,
3345                              // outputs: scan temp. reg, scan temp. reg
3346                              rscratch2, r13,
3347                              no_such_interface,
3348                              /*return_method=*/false);
3349 
3350   // profile this call
3351   __ profile_virtual_call(r3, r13, r19);
3352 
3353   // Get declaring interface class from method, and itable index
3354   __ ldr(r0, Address(rmethod, Method::const_offset()));
3355   __ ldr(r0, Address(r0, ConstMethod::constants_offset()));
3356   __ ldr(r0, Address(r0, ConstantPool::pool_holder_offset_in_bytes()));
3357   __ ldrw(rmethod, Address(rmethod, Method::itable_index_offset()));
3358   __ subw(rmethod, rmethod, Method::itable_index_max);
3359   __ negw(rmethod, rmethod);
3360 
3361   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3362                              r3, r0, rmethod,
3363                              // outputs: method, scan temp. reg
3364                              rmethod, r13,
3365                              no_such_interface);
3366 
3367   // rmethod,: methodOop to call
3368   // r2: receiver
3369   // Check for abstract method error
3370   // Note: This should be done more efficiently via a throw_abstract_method_error
3371   //       interpreter entry point and a conditional jump to it in case of a null
3372   //       method.
3373   __ cbz(rmethod, no_such_method);
3374 
3375   __ profile_arguments_type(r3, rmethod, r13, true);
3376 
3377   // do the call
3378   // r2: receiver
3379   // rmethod,: methodOop
3380   __ jump_from_interpreted(rmethod, r3);
3381   __ should_not_reach_here();
3382 
3383   // exception handling code follows...
3384   // note: must restore interpreter registers to canonical
3385   //       state for exception handling to work correctly!
3386 
3387   __ bind(no_such_method);
3388   // throw exception
3389   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3390   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3391   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
3392   // the call_VM checks for exception, so we should never return here.
3393   __ should_not_reach_here();
3394 
3395   __ bind(no_such_interface);
3396   // throw exception
3397   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3398   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3399   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3400                    InterpreterRuntime::throw_IncompatibleClassChangeError));
3401   // the call_VM checks for exception, so we should never return here.
3402   __ should_not_reach_here();
3403   return;
3404 }
3405 
3406 void TemplateTable::invokehandle(int byte_no) {
3407   transition(vtos, vtos);
3408   assert(byte_no == f1_byte, "use this argument");
3409 
3410   if (!EnableInvokeDynamic) {
3411     // rewriter does not generate this bytecode
3412     __ should_not_reach_here();
3413     return;
3414   }
3415 
3416   prepare_invoke(byte_no, rmethod, r0, r2);
3417   __ verify_method_ptr(r2);
3418   __ verify_oop(r2);
3419   __ null_check(r2);
3420 
3421   // FIXME: profile the LambdaForm also
3422 
3423   // r13 is safe to use here as a scratch reg because it is about to
3424   // be clobbered by jump_from_interpreted().
3425   __ profile_final_call(r13);
3426   __ profile_arguments_type(r13, rmethod, r4, true);
3427 
3428   __ jump_from_interpreted(rmethod, r0);
3429 }
3430 
3431 void TemplateTable::invokedynamic(int byte_no) {
3432   transition(vtos, vtos);
3433   assert(byte_no == f1_byte, "use this argument");
3434 
3435   if (!EnableInvokeDynamic) {
3436     // We should not encounter this bytecode if !EnableInvokeDynamic.
3437     // The verifier will stop it.  However, if we get past the verifier,
3438     // this will stop the thread in a reasonable way, without crashing the JVM.
3439     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3440                      InterpreterRuntime::throw_IncompatibleClassChangeError));
3441     // the call_VM checks for exception, so we should never return here.
3442     __ should_not_reach_here();
3443     return;
3444   }
3445 
3446   prepare_invoke(byte_no, rmethod, r0);
3447 
3448   // r0: CallSite object (from cpool->resolved_references[])
3449   // rmethod: MH.linkToCallSite method (from f2)
3450 
3451   // Note:  r0_callsite is already pushed by prepare_invoke
3452 
3453   // %%% should make a type profile for any invokedynamic that takes a ref argument
3454   // profile this call
3455   __ profile_call(rbcp);
3456   __ profile_arguments_type(r3, rmethod, r13, false);
3457 
3458   __ verify_oop(r0);
3459 
3460   __ jump_from_interpreted(rmethod, r0);
3461 }
3462 
3463 
3464 //-----------------------------------------------------------------------------
3465 // Allocation
3466 
3467 void TemplateTable::_new() {
3468   transition(vtos, atos);
3469 
3470   __ get_unsigned_2_byte_index_at_bcp(r3, 1);
3471   Label slow_case;
3472   Label done;
3473   Label initialize_header;
3474   Label initialize_object; // including clearing the fields
3475   Label allocate_shared;
3476 
3477   __ get_cpool_and_tags(r4, r0);
3478   // Make sure the class we're about to instantiate has been resolved.
3479   // This is done before loading InstanceKlass to be consistent with the order
3480   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3481   const int tags_offset = Array<u1>::base_offset_in_bytes();
3482   __ lea(rscratch1, Address(r0, r3, Address::lsl(0)));
3483   __ lea(rscratch1, Address(rscratch1, tags_offset));
3484   __ ldarb(rscratch1, rscratch1);
3485   __ cmp(rscratch1, JVM_CONSTANT_Class);
3486   __ br(Assembler::NE, slow_case);
3487 
3488   // get InstanceKlass
3489   __ lea(r4, Address(r4, r3, Address::lsl(3)));
3490   __ ldr(r4, Address(r4, sizeof(ConstantPool)));
3491 
3492   // make sure klass is initialized & doesn't have finalizer
3493   // make sure klass is fully initialized
3494   __ ldrb(rscratch1, Address(r4, InstanceKlass::init_state_offset()));
3495   __ cmp(rscratch1, InstanceKlass::fully_initialized);
3496   __ br(Assembler::NE, slow_case);
3497 
3498   // get instance_size in InstanceKlass (scaled to a count of bytes)
3499   __ ldrw(r3,
3500           Address(r4,
3501                   Klass::layout_helper_offset()));
3502   // test to see if it has a finalizer or is malformed in some way
3503   __ tbnz(r3, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
3504 
3505   // Allocate the instance
3506   // 1) Try to allocate in the TLAB
3507   // 2) if fail and the object is large allocate in the shared Eden
3508   // 3) if the above fails (or is not applicable), go to a slow case
3509   // (creates a new TLAB, etc.)
3510 
3511   const bool allow_shared_alloc =
3512     Universe::heap()->supports_inline_contig_alloc() && !CMSIncrementalMode;
3513 
3514   if (UseTLAB) {
3515     __ tlab_allocate(r0, r3, 0, noreg, r1,
3516                      allow_shared_alloc ? allocate_shared : slow_case);
3517 
3518     if (ZeroTLAB) {
3519       // the fields have been already cleared
3520       __ b(initialize_header);
3521     } else {
3522       // initialize both the header and fields
3523       __ b(initialize_object);
3524     }
3525   }
3526 
3527   // Allocation in the shared Eden, if allowed.
3528   //
3529   // r3: instance size in bytes
3530   if (allow_shared_alloc) {
3531     __ bind(allocate_shared);
3532 
3533     __ eden_allocate(r0, r3, 0, r10, slow_case);
3534     __ incr_allocated_bytes(rthread, r3, 0, rscratch1);
3535   }
3536 
3537   if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) {
3538     // The object is initialized before the header.  If the object size is
3539     // zero, go directly to the header initialization.
3540     __ bind(initialize_object);
3541     __ sub(r3, r3, sizeof(oopDesc));
3542     __ cbz(r3, initialize_header);
3543 
3544     // Initialize object fields
3545     {
3546       __ add(r2, r0, sizeof(oopDesc));
3547       Label loop;
3548       __ bind(loop);
3549       __ str(zr, Address(__ post(r2, BytesPerLong)));
3550       __ sub(r3, r3, BytesPerLong);
3551       __ cbnz(r3, loop);
3552     }
3553 
3554     // initialize object header only.
3555     __ bind(initialize_header);
3556     if (UseBiasedLocking) {
3557       __ ldr(rscratch1, Address(r4, Klass::prototype_header_offset()));
3558     } else {
3559       __ mov(rscratch1, (intptr_t)markOopDesc::prototype());
3560     }
3561     __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
3562     __ store_klass_gap(r0, zr);  // zero klass gap for compressed oops
3563     __ store_klass(r0, r4);      // store klass last
3564 
3565     {
3566       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3567       // Trigger dtrace event for fastpath
3568       __ push(atos); // save the return value
3569       __ call_VM_leaf(
3570            CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), r0);
3571       __ pop(atos); // restore the return value
3572 
3573     }
3574     __ b(done);
3575   }
3576 
3577   // slow case
3578   __ bind(slow_case);
3579   __ get_constant_pool(c_rarg1);
3580   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3581   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3582   __ verify_oop(r0);
3583 
3584   // continue
3585   __ bind(done);
3586   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3587   __ membar(Assembler::StoreStore);
3588 }
3589 
3590 void TemplateTable::newarray() {
3591   transition(itos, atos);
3592   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3593   __ mov(c_rarg2, r0);
3594   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3595           c_rarg1, c_rarg2);
3596   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3597   __ membar(Assembler::StoreStore);
3598 }
3599 
3600 void TemplateTable::anewarray() {
3601   transition(itos, atos);
3602   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3603   __ get_constant_pool(c_rarg1);
3604   __ mov(c_rarg3, r0);
3605   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3606           c_rarg1, c_rarg2, c_rarg3);
3607   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3608   __ membar(Assembler::StoreStore);
3609 }
3610 
3611 void TemplateTable::arraylength() {
3612   transition(atos, itos);
3613   __ null_check(r0, arrayOopDesc::length_offset_in_bytes());
3614   __ ldrw(r0, Address(r0, arrayOopDesc::length_offset_in_bytes()));
3615 }
3616 
3617 void TemplateTable::checkcast()
3618 {
3619   transition(atos, atos);
3620   Label done, is_null, ok_is_subtype, quicked, resolved;
3621   __ cbz(r0, is_null);
3622 
3623   // Get cpool & tags index
3624   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3625   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3626   // See if bytecode has already been quicked
3627   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3628   __ lea(r1, Address(rscratch1, r19));
3629   __ ldarb(r1, r1);
3630   __ cmp(r1, JVM_CONSTANT_Class);
3631   __ br(Assembler::EQ, quicked);
3632 
3633   __ push(atos); // save receiver for result, and for GC
3634   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3635   // vm_result_2 has metadata result
3636   __ get_vm_result_2(r0, rthread);
3637   __ pop(r3); // restore receiver
3638   __ b(resolved);
3639 
3640   // Get superklass in r0 and subklass in r3
3641   __ bind(quicked);
3642   __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
3643   __ lea(r0, Address(r2, r19, Address::lsl(3)));
3644   __ ldr(r0, Address(r0, sizeof(ConstantPool)));
3645 
3646   __ bind(resolved);
3647   __ load_klass(r19, r3);
3648 
3649   // Generate subtype check.  Blows r2, r5.  Object in r3.
3650   // Superklass in r0.  Subklass in r19.
3651   __ gen_subtype_check(r19, ok_is_subtype);
3652 
3653   // Come here on failure
3654   __ push(r3);
3655   // object is at TOS
3656   __ b(Interpreter::_throw_ClassCastException_entry);
3657 
3658   // Come here on success
3659   __ bind(ok_is_subtype);
3660   __ mov(r0, r3); // Restore object in r3
3661 
3662   // Collect counts on whether this test sees NULLs a lot or not.
3663   if (ProfileInterpreter) {
3664     __ b(done);
3665     __ bind(is_null);
3666     __ profile_null_seen(r2);
3667   } else {
3668     __ bind(is_null);   // same as 'done'
3669   }
3670   __ bind(done);
3671 }
3672 
3673 void TemplateTable::instanceof() {
3674   transition(atos, itos);
3675   Label done, is_null, ok_is_subtype, quicked, resolved;
3676   __ cbz(r0, is_null);
3677 
3678   // Get cpool & tags index
3679   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3680   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3681   // See if bytecode has already been quicked
3682   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3683   __ lea(r1, Address(rscratch1, r19));
3684   __ ldarb(r1, r1);
3685   __ cmp(r1, JVM_CONSTANT_Class);
3686   __ br(Assembler::EQ, quicked);
3687 
3688   __ push(atos); // save receiver for result, and for GC
3689   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3690   // vm_result_2 has metadata result
3691   __ get_vm_result_2(r0, rthread);
3692   __ pop(r3); // restore receiver
3693   __ verify_oop(r3);
3694   __ load_klass(r3, r3);
3695   __ b(resolved);
3696 
3697   // Get superklass in r0 and subklass in r3
3698   __ bind(quicked);
3699   __ load_klass(r3, r0);
3700   __ lea(r0, Address(r2, r19, Address::lsl(3)));
3701   __ ldr(r0, Address(r0, sizeof(ConstantPool)));
3702 
3703   __ bind(resolved);
3704 
3705   // Generate subtype check.  Blows r2, r5
3706   // Superklass in r0.  Subklass in r3.
3707   __ gen_subtype_check(r3, ok_is_subtype);
3708 
3709   // Come here on failure
3710   __ mov(r0, 0);
3711   __ b(done);
3712   // Come here on success
3713   __ bind(ok_is_subtype);
3714   __ mov(r0, 1);
3715 
3716   // Collect counts on whether this test sees NULLs a lot or not.
3717   if (ProfileInterpreter) {
3718     __ b(done);
3719     __ bind(is_null);
3720     __ profile_null_seen(r2);
3721   } else {
3722     __ bind(is_null);   // same as 'done'
3723   }
3724   __ bind(done);
3725   // r0 = 0: obj == NULL or  obj is not an instanceof the specified klass
3726   // r0 = 1: obj != NULL and obj is     an instanceof the specified klass
3727 }
3728 
3729 //-----------------------------------------------------------------------------
3730 // Breakpoints
3731 void TemplateTable::_breakpoint() {
3732   // Note: We get here even if we are single stepping..
3733   // jbug inists on setting breakpoints at every bytecode
3734   // even if we are in single step mode.
3735 
3736   transition(vtos, vtos);
3737 
3738   // get the unpatched byte code
3739   __ get_method(c_rarg1);
3740   __ call_VM(noreg,
3741              CAST_FROM_FN_PTR(address,
3742                               InterpreterRuntime::get_original_bytecode_at),
3743              c_rarg1, rbcp);
3744   __ mov(r19, r0);
3745 
3746   // post the breakpoint event
3747   __ call_VM(noreg,
3748              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
3749              rmethod, rbcp);
3750 
3751   // complete the execution of original bytecode
3752   __ mov(rscratch1, r19);
3753   __ dispatch_only_normal(vtos);
3754 }
3755 
3756 //-----------------------------------------------------------------------------
3757 // Exceptions
3758 
3759 void TemplateTable::athrow() {
3760   transition(atos, vtos);
3761   __ null_check(r0);
3762   __ b(Interpreter::throw_exception_entry());
3763 }
3764 
3765 //-----------------------------------------------------------------------------
3766 // Synchronization
3767 //
3768 // Note: monitorenter & exit are symmetric routines; which is reflected
3769 //       in the assembly code structure as well
3770 //
3771 // Stack layout:
3772 //
3773 // [expressions  ] <--- esp               = expression stack top
3774 // ..
3775 // [expressions  ]
3776 // [monitor entry] <--- monitor block top = expression stack bot
3777 // ..
3778 // [monitor entry]
3779 // [frame data   ] <--- monitor block bot
3780 // ...
3781 // [saved rbp    ] <--- rbp
3782 void TemplateTable::monitorenter()
3783 {
3784   transition(atos, vtos);
3785 
3786   // check for NULL object
3787   __ null_check(r0);
3788 
3789   const Address monitor_block_top(
3790         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3791   const Address monitor_block_bot(
3792         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3793   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3794 
3795   Label allocated;
3796 
3797   // initialize entry pointer
3798   __ mov(c_rarg1, zr); // points to free slot or NULL
3799 
3800   // find a free slot in the monitor block (result in c_rarg1)
3801   {
3802     Label entry, loop, exit;
3803     __ ldr(c_rarg3, monitor_block_top); // points to current entry,
3804                                         // starting with top-most entry
3805     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3806 
3807     __ b(entry);
3808 
3809     __ bind(loop);
3810     // check if current entry is used
3811     // if not used then remember entry in c_rarg1
3812     __ ldr(rscratch1, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
3813     __ cmp(zr, rscratch1);
3814     __ csel(c_rarg1, c_rarg3, c_rarg1, Assembler::EQ);
3815     // check if current entry is for same object
3816     __ cmp(r0, rscratch1);
3817     // if same object then stop searching
3818     __ br(Assembler::EQ, exit);
3819     // otherwise advance to next entry
3820     __ add(c_rarg3, c_rarg3, entry_size);
3821     __ bind(entry);
3822     // check if bottom reached
3823     __ cmp(c_rarg3, c_rarg2);
3824     // if not at bottom then check this entry
3825     __ br(Assembler::NE, loop);
3826     __ bind(exit);
3827   }
3828 
3829   __ cbnz(c_rarg1, allocated); // check if a slot has been found and
3830                             // if found, continue with that on
3831 
3832   // allocate one if there's no free slot
3833   {
3834     Label entry, loop;
3835     // 1. compute new pointers            // rsp: old expression stack top
3836     __ ldr(c_rarg1, monitor_block_bot);   // c_rarg1: old expression stack bottom
3837     __ sub(esp, esp, entry_size);         // move expression stack top
3838     __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
3839     __ mov(c_rarg3, esp);                 // set start value for copy loop
3840     __ str(c_rarg1, monitor_block_bot);   // set new monitor block bottom
3841 
3842     __ sub(sp, sp, entry_size);           // make room for the monitor
3843 
3844     __ b(entry);
3845     // 2. move expression stack contents
3846     __ bind(loop);
3847     __ ldr(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
3848                                                    // word from old location
3849     __ str(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
3850     __ add(c_rarg3, c_rarg3, wordSize);            // advance to next word
3851     __ bind(entry);
3852     __ cmp(c_rarg3, c_rarg1);        // check if bottom reached
3853     __ br(Assembler::NE, loop);      // if not at bottom then
3854                                      // copy next word
3855   }
3856 
3857   // call run-time routine
3858   // c_rarg1: points to monitor entry
3859   __ bind(allocated);
3860 
3861   // Increment bcp to point to the next bytecode, so exception
3862   // handling for async. exceptions work correctly.
3863   // The object has already been poped from the stack, so the
3864   // expression stack looks correct.
3865   __ increment(rbcp);
3866 
3867   // store object
3868   __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3869   __ lock_object(c_rarg1);
3870 
3871   // check to make sure this monitor doesn't cause stack overflow after locking
3872   __ save_bcp();  // in case of exception
3873   __ generate_stack_overflow_check(0);
3874 
3875   // The bcp has already been incremented. Just need to dispatch to
3876   // next instruction.
3877   __ dispatch_next(vtos);
3878 }
3879 
3880 
3881 void TemplateTable::monitorexit()
3882 {
3883   transition(atos, vtos);
3884 
3885   // check for NULL object
3886   __ null_check(r0);
3887 
3888   const Address monitor_block_top(
3889         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3890   const Address monitor_block_bot(
3891         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3892   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3893 
3894   Label found;
3895 
3896   // find matching slot
3897   {
3898     Label entry, loop;
3899     __ ldr(c_rarg1, monitor_block_top); // points to current entry,
3900                                         // starting with top-most entry
3901     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3902                                         // of monitor block
3903     __ b(entry);
3904 
3905     __ bind(loop);
3906     // check if current entry is for same object
3907     __ ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3908     __ cmp(r0, rscratch1);
3909     // if same object then stop searching
3910     __ br(Assembler::EQ, found);
3911     // otherwise advance to next entry
3912     __ add(c_rarg1, c_rarg1, entry_size);
3913     __ bind(entry);
3914     // check if bottom reached
3915     __ cmp(c_rarg1, c_rarg2);
3916     // if not at bottom then check this entry
3917     __ br(Assembler::NE, loop);
3918   }
3919 
3920   // error handling. Unlocking was not block-structured
3921   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3922                    InterpreterRuntime::throw_illegal_monitor_state_exception));
3923   __ should_not_reach_here();
3924 
3925   // call run-time routine
3926   __ bind(found);
3927   __ push_ptr(r0); // make sure object is on stack (contract with oopMaps)
3928   __ unlock_object(c_rarg1);
3929   __ pop_ptr(r0); // discard object
3930 }
3931 
3932 
3933 // Wide instructions
3934 void TemplateTable::wide()
3935 {
3936   __ load_unsigned_byte(r19, at_bcp(1));
3937   __ mov(rscratch1, (address)Interpreter::_wentry_point);
3938   __ ldr(rscratch1, Address(rscratch1, r19, Address::uxtw(3)));
3939   __ br(rscratch1);
3940 }
3941 
3942 
3943 // Multi arrays
3944 void TemplateTable::multianewarray() {
3945   transition(vtos, atos);
3946   __ load_unsigned_byte(r0, at_bcp(3)); // get number of dimensions
3947   // last dim is on top of stack; we want address of first one:
3948   // first_addr = last_addr + (ndims - 1) * wordSize
3949   __ lea(c_rarg1, Address(esp, r0, Address::uxtw(3)));
3950   __ sub(c_rarg1, c_rarg1, wordSize);
3951   call_VM(r0,
3952           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
3953           c_rarg1);
3954   __ load_unsigned_byte(r1, at_bcp(3));
3955   __ lea(esp, Address(esp, r1, Address::uxtw(3)));
3956 }
3957 #endif // !CC_INTERP