1 /*
   2  * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2015, Linaro Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "interp_masm_aarch32.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "interpreter/interpreterRuntime.hpp"
  32 #include "interpreter/templateTable.hpp"
  33 #include "memory/universe.inline.hpp"
  34 #include "oops/method.hpp"
  35 #include "oops/methodData.hpp"
  36 #include "oops/objArrayKlass.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "prims/methodHandles.hpp"
  39 #include "runtime/sharedRuntime.hpp"
  40 #include "runtime/stubRoutines.hpp"
  41 #include "runtime/synchronizer.hpp"
  42 #include "vm_version_aarch32.hpp"
  43 
  44 #ifndef CC_INTERP
  45 
  46 #define __ _masm->
  47 
  48 // FIXME Remove both of these functions
  49 void print_constantpoolcache(ConstantPoolCache *cpc) {
  50   cpc->print_on(tty);
  51 }
  52 void print_constantpool(ConstantPool *cp) {
  53   cp->print_on(tty);
  54 }
  55 // FIXME They're just for debugging
  56 
  57 // Platform-dependent initialization
  58 
  59 extern void aarch32TestHook();
  60 
  61 void TemplateTable::pd_initialize() {
  62   aarch32TestHook();
  63 }
  64 
  65 // Address computation: local variables
  66 
  67 static inline Address iaddress(int n) {
  68   return Address(rlocals, Interpreter::local_offset_in_bytes(n));
  69 }
  70 
  71 static inline Address laddress(int n) {
  72   return iaddress(n + 1);
  73 }
  74 
  75 static inline Address faddress(int n) {
  76   return iaddress(n);
  77 }
  78 
  79 static inline Address daddress(int n) {
  80   return laddress(n);
  81 }
  82 
  83 static inline Address aaddress(int n) {
  84   return iaddress(n);
  85 }
  86 
  87 static inline Address iaddress(Register r) {
  88   // FIXME
  89   return Address(rlocals, r, lsl(2));
  90 }
  91 
  92 // Note these two are different as VLDR/VSTR don't
  93 // support base + (offset{ << x })
  94 static inline Address faddress(Register r, Register scratch,
  95                                InterpreterMacroAssembler* _masm) {
  96   __ lea(scratch, Address(rlocals, r, lsl(2)));
  97   return Address(scratch);
  98 }
  99 
 100 static inline Address daddress(Register r, Register scratch,
 101                                InterpreterMacroAssembler* _masm) {
 102   __ lea(scratch, Address(rlocals, r, lsl(2)));
 103   return Address(scratch, Interpreter::local_offset_in_bytes(1));
 104 }
 105 
 106 static inline Address laddress(Register r, Register scratch,
 107                                InterpreterMacroAssembler * _masm) {
 108   return daddress(r, scratch, _masm);
 109 }
 110 
 111 static inline Address aaddress(Register r) {
 112   return iaddress(r);
 113 }
 114 
 115 static inline Address at_rsp() {
 116   return Address(sp, 0);
 117 }
 118 
 119 // At top of Java expression stack which may be different than sp().  It
 120 // isn't for category 1 objects.
 121 static inline Address at_tos   () {
 122   return Address(sp,  Interpreter::expr_offset_in_bytes(0));
 123 }
 124 
 125 static inline Address at_tos_p1() {
 126   return Address(sp,  Interpreter::expr_offset_in_bytes(1));
 127 }
 128 
 129 static inline Address at_tos_p2() {
 130   return Address(sp,  Interpreter::expr_offset_in_bytes(2));
 131 }
 132 
 133 static inline Address at_tos_p3() {
 134   return Address(sp,  Interpreter::expr_offset_in_bytes(3));
 135 }
 136 
 137 static inline Address at_tos_p4() {
 138   return Address(sp,  Interpreter::expr_offset_in_bytes(4));
 139 }
 140 
 141 static inline Address at_tos_p5() {
 142   return Address(sp,  Interpreter::expr_offset_in_bytes(5));
 143 }
 144 
 145 // Condition conversion
 146 static Assembler::Condition j_not(TemplateTable::Condition cc) {
 147   switch (cc) {
 148   case TemplateTable::equal        : return Assembler::NE;
 149   case TemplateTable::not_equal    : return Assembler::EQ;
 150   case TemplateTable::less         : return Assembler::GE;
 151   case TemplateTable::less_equal   : return Assembler::GT;
 152   case TemplateTable::greater      : return Assembler::LE;
 153   case TemplateTable::greater_equal: return Assembler::LT;
 154   }
 155   ShouldNotReachHere();
 156   return Assembler::EQ;
 157 }
 158 
 159 
 160 // Miscelaneous helper routines
 161 // Store an oop (or NULL) at the Address described by obj.
 162 // If val == noreg this means store a NULL
 163 static void do_oop_store(InterpreterMacroAssembler* _masm,
 164                          Address obj,
 165                          Register val,
 166                          BarrierSet::Name barrier,
 167                          bool precise) {
 168   assert(val == noreg || val == r0, "parameter is just for looks");
 169   switch (barrier) {
 170 #if INCLUDE_ALL_GCS
 171     case BarrierSet::G1SATBCTLogging:
 172       {
 173         // flatten object address if needed
 174         if (obj.index() == noreg && obj.offset() == 0) {
 175           if (obj.base() != r3) {
 176             __ mov(r3, obj.base());
 177           }
 178         } else {
 179           __ lea(r3, obj);
 180         }
 181         __ g1_write_barrier_pre(r3 /* obj */,
 182                                 r1 /* pre_val */,
 183                                 rthread /* thread */,
 184                                 r14  /* tmp */,
 185                                 val != noreg /* tosca_live */,
 186                                 false /* expand_call */);
 187         if (val == noreg) {
 188           __ store_heap_oop_null(Address(r3, 0));
 189         } else {
 190           // G1 barrier needs uncompressed oop for region cross check.
 191           Register new_val = val;
 192           __ store_heap_oop(Address(r3, 0), val);
 193           __ g1_write_barrier_post(r3 /* store_adr */,
 194                                    new_val /* new_val */,
 195                                    rthread /* thread */,
 196                                    r14 /* tmp */,
 197                                    r1 /* tmp2 */);
 198         }
 199 
 200       }
 201       break;
 202 #endif // INCLUDE_ALL_GCS
 203     case BarrierSet::CardTableModRef:
 204     case BarrierSet::CardTableExtension:
 205       {
 206         if (val == noreg) {
 207           __ store_heap_oop_null(obj);
 208         } else {
 209           __ store_heap_oop(obj, val);
 210           // flatten object address if needed
 211           if (!precise || (obj.index() == noreg && obj.offset() == 0)) {
 212             __ store_check(obj.base());
 213           } else {
 214             __ lea(r3, obj);
 215             __ store_check(r3);
 216           }
 217         }
 218       }
 219       break;
 220     case BarrierSet::ModRef:
 221     case BarrierSet::Other:
 222       if (val == noreg) {
 223         __ store_heap_oop_null(obj);
 224       } else {
 225         __ store_heap_oop(obj, val);
 226       }
 227       break;
 228     default      :
 229       ShouldNotReachHere();
 230 
 231   }
 232 }
 233 
 234 Address TemplateTable::at_bcp(int offset) {
 235   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 236   return Address(rbcp, offset);
 237 }
 238 
 239 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 240                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 241                                    int byte_no)
 242 {
 243   if (!RewriteBytecodes)  return;
 244   Label L_patch_done;
 245 
 246   switch (bc) {
 247   case Bytecodes::_fast_aputfield:
 248   case Bytecodes::_fast_bputfield:
 249   case Bytecodes::_fast_zputfield:
 250   case Bytecodes::_fast_cputfield:
 251   case Bytecodes::_fast_dputfield:
 252   case Bytecodes::_fast_fputfield:
 253   case Bytecodes::_fast_iputfield:
 254   case Bytecodes::_fast_lputfield:
 255   case Bytecodes::_fast_sputfield:
 256     {
 257       // We skip bytecode quickening for putfield instructions when
 258       // the put_code written to the constant pool cache is zero.
 259       // This is required so that every execution of this instruction
 260       // calls out to InterpreterRuntime::resolve_get_put to do
 261       // additional, required work.
 262       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 263       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 264       __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
 265       __ mov(bc_reg, bc);
 266       __ cmp(temp_reg, (unsigned) 0);
 267       __ b(L_patch_done, Assembler::EQ);  // don't patch
 268     }
 269     break;
 270   default:
 271     assert(byte_no == -1, "sanity");
 272     // the pair bytecodes have already done the load.
 273     if (load_bc_into_bc_reg) {
 274       __ mov(bc_reg, bc);
 275     }
 276   }
 277 
 278   if (JvmtiExport::can_post_breakpoint()) {
 279     Label L_fast_patch;
 280     // if a breakpoint is present we can't rewrite the stream directly
 281     __ load_unsigned_byte(temp_reg, at_bcp(0));
 282     __ cmp(temp_reg, Bytecodes::_breakpoint);
 283     __ b(L_fast_patch, Assembler::NE);
 284     // Let breakpoint table handling rewrite to quicker bytecode
 285     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), rmethod, rbcp, bc_reg);
 286     __ b(L_patch_done);
 287     __ bind(L_fast_patch);
 288   }
 289 
 290 #ifdef ASSERT
 291   Label L_okay;
 292   __ load_unsigned_byte(temp_reg, at_bcp(0));
 293   __ cmp(temp_reg, (int) Bytecodes::java_code(bc));
 294   __ b(L_okay, Assembler::EQ);
 295   __ cmp(temp_reg, bc_reg);
 296   __ b(L_okay, Assembler::EQ);
 297   __ stop("patching the wrong bytecode");
 298   __ bind(L_okay);
 299 #endif
 300 
 301   // patch bytecode
 302   __ strb(bc_reg, at_bcp(0));
 303   __ bind(L_patch_done);
 304 }
 305 
 306 
 307 // Individual instructions
 308 
 309 void TemplateTable::nop() {
 310   transition(vtos, vtos);
 311   // nothing to do
 312 }
 313 
 314 void TemplateTable::shouldnotreachhere() {
 315   transition(vtos, vtos);
 316   __ stop("shouldnotreachhere bytecode");
 317 }
 318 
 319 void TemplateTable::aconst_null()
 320 {
 321   transition(vtos, atos);
 322   __ mov(r0, 0);
 323 }
 324 
 325 void TemplateTable::iconst(int value)
 326 {
 327   transition(vtos, itos);
 328   __ mov(r0, value);
 329 }
 330 
 331 void TemplateTable::lconst(int value)
 332 {
 333   // int is 32 bit and only ever used for loading small values
 334   __ mov(r0, value & 0xffffffff);
 335   __ mov(r1, 0);
 336 }
 337 
 338 void TemplateTable::fconst(int value)
 339 {
 340   transition(vtos, ftos);
 341   float fval = value;
 342   assert(value == 0 || value == 1 || value == 2, "invalid float const");
 343   if (hasFPU()) {
 344     if(__ operand_valid_for_float_immediate(fval)) {
 345       __ vmov_f32(d0, fval);
 346     } else {
 347       __ mov(r0, *((uint32_t*)&fval));
 348       __ vmov_f32(d0, r0);
 349     }
 350   } else {
 351     __ mov(r0, *((uint32_t*)&fval));
 352   }
 353 }
 354 
 355 void TemplateTable::dconst(int value)
 356 {
 357   transition(vtos, dtos);
 358   double dval = value;
 359   assert(value == 0 || value == 1 || value == 2, "invalid double const");
 360   if (hasFPU()) {
 361     if(__ operand_valid_for_double_immediate(dval)) {
 362       __ vmov_f64(d0, dval);
 363     } else {
 364       uint32_t* ptr = (uint32_t*)&dval;
 365       __ mov(r0, *ptr);
 366       __ mov(r1, *(ptr + 1));
 367       __ vmov_f64(d0, r0, r1);
 368     }
 369   } else {
 370     uint32_t* ptr = (uint32_t*)&dval;
 371     __ mov(r0, *ptr);
 372     __ mov(r1, *(ptr + 1));
 373   }
 374 }
 375 
 376 void TemplateTable::bipush()
 377 {
 378   transition(vtos, itos);
 379   __ load_signed_byte(r0, at_bcp(1));
 380 }
 381 
 382 void TemplateTable::sipush()
 383 {
 384   transition(vtos, itos);
 385   __ load_unsigned_short(r0, at_bcp(1));
 386   __ rev(r0, r0);
 387   __ asr(r0, r0, 16);
 388 }
 389 
 390 void TemplateTable::ldc(bool wide)
 391 {
 392   transition(vtos, vtos);
 393   Label call_ldc, notFloat, notClass, Done;
 394 
 395   if (wide) {
 396     __ get_unsigned_2_byte_index_at_bcp(r1, 1);
 397   } else {
 398     __ load_unsigned_byte(r1, at_bcp(1));
 399   }
 400   __ get_cpool_and_tags(r2, r0);
 401 
 402   const int base_offset = ConstantPool::header_size() * wordSize;
 403   const int tags_offset = Array<u1>::base_offset_in_bytes();
 404 
 405   // get type
 406   __ add(r3, r1, tags_offset);
 407   __ ldrb(r3, Address(r0, r3));
 408 
 409   // unresolved class - get the resolved class
 410   __ cmp(r3, JVM_CONSTANT_UnresolvedClass);
 411   __ b(call_ldc, Assembler::EQ);
 412 
 413   // unresolved class in error state - call into runtime to throw the error
 414   // from the first resolution attempt
 415   __ cmp(r3, JVM_CONSTANT_UnresolvedClassInError);
 416   __ b(call_ldc, Assembler::EQ);
 417 
 418   // resolved class - need to call vm to get java mirror of the class
 419   __ cmp(r3, JVM_CONSTANT_Class);
 420   __ b(notClass, Assembler::NE);
 421 
 422   __ bind(call_ldc);
 423   __ mov(c_rarg1, wide);
 424   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
 425   __ push_ptr(r0);
 426   __ verify_oop(r0);
 427   __ b(Done);
 428 
 429   __ bind(notClass);
 430   if (hasFPU()) {
 431     __ cmp(r3, JVM_CONSTANT_Float);
 432     __ b(notFloat, Assembler::NE);
 433     // ftos
 434     __ adds(r1, r2, r1, lsl(2));
 435     __ vldr_f32(d0, Address(r1, base_offset));
 436 
 437     __ push_f();
 438 
 439     __ b(Done);
 440 
 441     __ bind(notFloat);
 442   } else {
 443         // Soft FP pass through T_INT case.
 444 #ifdef ASSERT
 445         __ cmp(r3, JVM_CONSTANT_Float);
 446         __ mov(r3, JVM_CONSTANT_Integer,  Assembler::EQ);
 447 #endif // ASSER
 448   }
 449 #ifdef ASSERT
 450   {
 451     Label L;
 452     __ cmp(r3, JVM_CONSTANT_Integer);
 453     __ b(L, Assembler::EQ);
 454     // String and Object are rewritten to fast_aldc
 455     __ stop("unexpected tag type in ldc");
 456     __ bind(L);
 457   }
 458 #endif
 459   // itos JVM_CONSTANT_Integer only
 460   __ adds(r1, r2, r1, lsl(2));
 461   __ ldr(r0, Address(r1, base_offset));
 462   __ push_i(r0);
 463   __ bind(Done);
 464 }
 465 
 466 // Fast path for caching oop constants.
 467 void TemplateTable::fast_aldc(bool wide)
 468 {
 469   transition(vtos, atos);
 470 
 471   Register result = r0;
 472   Register tmp = r1;
 473   int index_size = wide ? sizeof(u2) : sizeof(u1);
 474 
 475   Label resolved;
 476 
 477   // We are resolved if the resolved reference cache entry contains a
 478   // non-null object (String, MethodType, etc.)
 479   assert_different_registers(result, tmp);
 480   __ get_cache_index_at_bcp(tmp, 1, index_size);
 481   __ load_resolved_reference_at_index(result, tmp);
 482   __ cbnz(result, resolved);
 483 
 484   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 485 
 486   // first time invocation - must resolve first
 487   __ mov(tmp, (int)bytecode());
 488   __ call_VM(result, entry, tmp);
 489 
 490   __ bind(resolved);
 491 
 492   if (VerifyOops) {
 493     __ verify_oop(result);
 494   }
 495 }
 496 
 497 void TemplateTable::ldc2_w()
 498 {
 499   transition(vtos, vtos);
 500    Label Done;
 501   __ get_unsigned_2_byte_index_at_bcp(r0, 1);
 502 
 503   __ get_cpool_and_tags(r1, r2);
 504   const int base_offset = ConstantPool::header_size() * wordSize;
 505   const int tags_offset = Array<u1>::base_offset_in_bytes();
 506 
 507   // get type
 508   __ lea(r2, Address(r2, r0, lsl(0)));
 509   __ load_unsigned_byte(r2, Address(r2, tags_offset));
 510   if (hasFPU()) {
 511     Label Long;
 512     __ cmp(r2, (int)JVM_CONSTANT_Double);
 513     __ b(Long, Assembler::NE);
 514     // dtos
 515     __ lea (r2, Address(r1, r0, lsl(2)));
 516     __ vldr_f64(d0, Address(r2, base_offset));
 517     __ push_d();
 518     __ b(Done);
 519 
 520     __ bind(Long);
 521   }
 522   // ltos
 523   __ lea(r1, Address(r1, r0, lsl(2)));
 524   __ ldr(r0, Address(r1, base_offset));
 525   __ ldr(r1, Address(r1, base_offset + wordSize));
 526   __ push_l();
 527 
 528   __ bind(Done);
 529 }
 530 
 531 void TemplateTable::locals_index(Register reg, int offset)
 532 {
 533   __ ldrb(reg, at_bcp(offset));
 534   __ neg(reg, reg);
 535 }
 536 
 537 void TemplateTable::iload() {
 538   transition(vtos, itos);
 539   if (RewriteFrequentPairs) {
 540     Label rewrite, done;
 541     Register bc = r2;
 542 
 543     // get next bytecode
 544     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
 545 
 546     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 547     // last two iloads in a pair.  Comparing against fast_iload means that
 548     // the next bytecode is neither an iload or a caload, and therefore
 549     // an iload pair.
 550     __ cmp(r1, Bytecodes::_iload);
 551     __ b(done, Assembler::EQ);
 552 
 553     // if _fast_iload rewrite to _fast_iload2
 554     __ cmp(r1, Bytecodes::_fast_iload);
 555     __ mov(bc, Bytecodes::_fast_iload2);
 556     __ b(rewrite, Assembler::EQ);
 557 
 558     // if _caload rewrite to _fast_icaload
 559     __ cmp(r1, Bytecodes::_caload);
 560     __ mov(bc, Bytecodes::_fast_icaload);
 561     __ b(rewrite, Assembler::EQ);
 562 
 563     // else rewrite to _fast_iload
 564     __ mov(bc, Bytecodes::_fast_iload);
 565 
 566     // rewrite
 567     // bc: new bytecode
 568     __ bind(rewrite);
 569     patch_bytecode(Bytecodes::_iload, bc, r1, false);
 570     __ bind(done);
 571 
 572   }
 573 
 574   // do iload, get the local value into tos
 575   locals_index(r1);
 576   __ ldr(r0, iaddress(r1));
 577   __ reg_printf("iloaded value %d\n", r0);
 578 }
 579 
 580 void TemplateTable::fast_iload2()
 581 {
 582   transition(vtos, itos);
 583   locals_index(r1);
 584   __ ldr(r0, iaddress(r1));
 585   __ push(itos);
 586   locals_index(r1, 3);
 587   __ ldr(r0, iaddress(r1));
 588 }
 589 
 590 void TemplateTable::fast_iload()
 591 {
 592   transition(vtos, itos);
 593   locals_index(r1);
 594   __ ldr(r0, iaddress(r1));
 595 }
 596 
 597 void TemplateTable::lload()
 598 {
 599   transition(vtos, ltos);
 600   locals_index(r2);
 601   __ ldrd(r0, r1, laddress(r2, r3, _masm));
 602 }
 603 
 604 void TemplateTable::fload()
 605 {
 606   transition(vtos, ftos);
 607   locals_index(r1);
 608   if (hasFPU()) {
 609       __ vldr_f32(d0, faddress(r1, r2, _masm));
 610   } else {
 611     __ ldr(r0, faddress(r1, r2, _masm));
 612   }
 613 }
 614 
 615 void TemplateTable::dload()
 616 {
 617   transition(vtos, dtos);
 618   if (hasFPU()) {
 619     __ ldrb(r1, at_bcp(1));
 620     __ sub(r1, rlocals, r1, lsl(LogBytesPerWord));
 621     __ vldr_f64(d0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 622   } else {
 623     locals_index(r2);
 624     __ ldrd(r0, r1, daddress(r2, r3, _masm));
 625   }
 626 }
 627 
 628 void TemplateTable::aload()
 629 {
 630   transition(vtos, atos);
 631   locals_index(r1);
 632   __ ldr(r0, iaddress(r1));
 633 }
 634 
 635 void TemplateTable::locals_index_wide(Register reg) {
 636   __ ldrh(reg, at_bcp(2));
 637   __ rev16(reg, reg);
 638   __ neg(reg, reg);
 639 }
 640 
 641 void TemplateTable::wide_iload() {
 642   transition(vtos, itos);
 643   locals_index_wide(r1);
 644   __ ldr(r0, iaddress(r1));
 645 }
 646 
 647 void TemplateTable::wide_lload()
 648 {
 649   transition(vtos, ltos);
 650   locals_index_wide(r2);
 651   __ ldrd(r0, r1, laddress(r2, r3, _masm));
 652 }
 653 
 654 void TemplateTable::wide_fload()
 655 {
 656   transition(vtos, ftos);
 657   locals_index_wide(r1);
 658   if (hasFPU()) {
 659       __ vldr_f32(d0, faddress(r1, rscratch1, _masm));
 660   } else {
 661   __ ldr (r0, faddress(r1, rscratch1, _masm));
 662   }
 663 }
 664 
 665 void TemplateTable::wide_dload()
 666 {
 667   transition(vtos, dtos);
 668   if (hasFPU()) {
 669     __ ldrh(r1, at_bcp(2));
 670     __ rev16(r1, r1);
 671     __ sub(r1, rlocals, r1, lsl(LogBytesPerWord));
 672     __ vldr_f64(d0, Address(r1, Interpreter::local_offset_in_bytes(1)));
 673   } else {
 674     locals_index_wide(r2);
 675     __ ldrd(r0, r1, daddress(r2, r3, _masm));
 676   }
 677 }
 678 
 679 void TemplateTable::wide_aload()
 680 {
 681   transition(vtos, atos);
 682   locals_index_wide(r1);
 683   __ ldr(r0, aaddress(r1));
 684 }
 685 
 686 void TemplateTable::index_check(Register array, Register index)
 687 {
 688   // destroys rscratch1
 689   // check array
 690   __ null_check(array, arrayOopDesc::length_offset_in_bytes());
 691   // sign extend index for use by indexed load
 692   // __ movl2ptr(index, index);
 693   // check index
 694   Register length = rscratch1;
 695   __ ldr(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 696   __ reg_printf("Checking index in array, array = %p, alen = %d, index = %d\n", array, length, index);
 697   __ cmp(index, length);
 698   if (index != r2) {
 699     // ??? convention: move aberrant index into r2 for exception message
 700     assert(r2 != array, "different registers");
 701     __ mov(r2, index);
 702   }
 703   Label ok;
 704   __ b(ok, Assembler::LO);
 705   __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 706   __ b(rscratch1);
 707   __ bind(ok);
 708 }
 709 
 710 void TemplateTable::iaload()
 711 {
 712   transition(itos, itos);
 713   __ mov(r2, r0);
 714   __ pop_ptr(r0);
 715   // r0: array
 716   // r2: index
 717   index_check(r0, r2); // leaves index in r2, kills rscratch1
 718   __ lea(r2, Address(r0, r2, lsl(2)));
 719   __ ldr(r0, Address(r2, arrayOopDesc::base_offset_in_bytes(T_INT)));
 720 }
 721 
 722 void TemplateTable::laload()
 723 {
 724   transition(itos, ltos);
 725   __ mov(r2, r0);
 726   __ pop_ptr(r0);
 727   // r0: array
 728   // r2: index
 729   index_check(r0, r2); // leaves index in r2, kills rscratch1
 730   __ lea(r2, Address(r0, r2, lsl(3)));
 731   __ lea(r2, Address(r2,  arrayOopDesc::base_offset_in_bytes(T_LONG)));
 732   __ atomic_ldrd(r0, r1, r2);
 733 }
 734 
 735 void TemplateTable::faload()
 736 {
 737   transition(itos, ftos);
 738   __ mov(r2, r0);
 739   __ pop_ptr(r0);
 740   // r0: array
 741   // r2: index
 742   index_check(r0, r2); // leaves index in r2, kills rscratch1
 743   __ lea(r2,  Address(r0, r2, lsl(2)));
 744   if (hasFPU()) {
 745       __ vldr_f32(d0, Address(r2,  arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
 746   } else {
 747     __ ldr(r0, Address(r2,  arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
 748   }
 749 }
 750 
 751 void TemplateTable::daload()
 752 {
 753   transition(itos, dtos);
 754   __ mov(r2, r0);
 755   __ pop_ptr(r0);
 756   // r0: array
 757   // r2: index
 758   index_check(r0, r2); // leaves index in r2, kills rscratch1
 759   __ lea(r2,  Address(r0, r2, lsl(3)));
 760   __ add(r2, r2, arrayOopDesc::base_offset_in_bytes(T_DOUBLE));
 761   __ atomic_ldrd(r0, r1, r2);
 762   if (hasFPU()) {
 763       __ vmov_f64(d0, r0, r1);
 764   }
 765 }
 766 
 767 void TemplateTable::aaload()
 768 {
 769   transition(itos, atos);
 770   __ mov(r2, r0);
 771   __ pop_ptr(r0);
 772   // r0: array
 773   // r2: index
 774   index_check(r0, r2); // leaves index in r2, kills rscratch1
 775   __ lea(r2, Address(r0, r2, lsl(2)));
 776   __ load_heap_oop(r0, Address(r2, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
 777 }
 778 
 779 void TemplateTable::baload()
 780 {
 781   transition(itos, itos);
 782   __ mov(r2, r0);
 783   __ pop_ptr(r0);
 784   // r0: array
 785   // r2: index
 786   index_check(r0, r2); // leaves index in r2, kills rscratch1
 787   __ lea(r2,  Address(r0, r2, lsl(0)));
 788   __ load_signed_byte(r0, Address(r2,  arrayOopDesc::base_offset_in_bytes(T_BYTE)));
 789 }
 790 
 791 void TemplateTable::caload()
 792 {
 793   transition(itos, itos);
 794   __ mov(r2, r0);
 795   __ pop_ptr(r0);
 796   // r0: array
 797   // r2: index
 798   index_check(r0, r2); // leaves index in r2, kills rscratch1
 799   __ lea(r2,  Address(r0, r2, lsl(1)));
 800   __ load_unsigned_short(r0, Address(r2,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 801 }
 802 
 803 // iload followed by caload frequent pair
 804 void TemplateTable::fast_icaload()
 805 {
 806   transition(vtos, itos);
 807   // load index out of locals
 808   locals_index(r2);
 809   __ ldr(r2, iaddress(r2));
 810 
 811   __ pop_ptr(r0);
 812 
 813   // r0: array
 814   // r2: index
 815   index_check(r0, r2); // leaves index in r1, kills rscratch1
 816   __ lea(r2,  Address(r0, r2, lsl(1)));
 817   __ load_unsigned_short(r0, Address(r2,  arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 818 }
 819 
 820 void TemplateTable::saload()
 821 {
 822   transition(itos, itos);
 823   __ mov(r2, r0);
 824   __ pop_ptr(r0);
 825   // r0: array
 826   // r2: index
 827   index_check(r0, r2); // leaves index in r2, kills rscratch1
 828   __ lea(r2,  Address(r0, r2, lsl(1)));
 829   __ load_signed_short(r0, Address(r2,  arrayOopDesc::base_offset_in_bytes(T_SHORT)));
 830 }
 831 
 832 void TemplateTable::iload(int n)
 833 {
 834   transition(vtos, itos);
 835   __ ldr(r0, iaddress(n));
 836 }
 837 
 838 void TemplateTable::lload(int n)
 839 {
 840   transition(vtos, ltos);
 841   __ ldrd(r0, r1, laddress(n));
 842 }
 843 
 844 void TemplateTable::fload(int n)
 845 {
 846   transition(vtos, ftos);
 847   if (hasFPU()) {
 848       __ vldr_f32(d0, faddress(n));
 849   } else {
 850     __ ldr(r0, faddress(n));
 851   }
 852 }
 853 
 854 void TemplateTable::dload(int n)
 855 {
 856   transition(vtos, dtos);
 857   if (hasFPU()) {
 858     __ vldr_f64(d0, daddress(n));
 859   } else {
 860     __ ldrd(r0, r1, daddress(n));
 861   }
 862 }
 863 
 864 void TemplateTable::aload(int n)
 865 {
 866   transition(vtos, atos);
 867   __ ldr(r0, iaddress(n));
 868   __ reg_printf("aload, loaded %p\n", r0);
 869 }
 870 
 871 void TemplateTable::aload_0() {
 872   // According to bytecode histograms, the pairs:
 873   //
 874   // _aload_0, _fast_igetfield
 875   // _aload_0, _fast_agetfield
 876   // _aload_0, _fast_fgetfield
 877   //
 878   // occur frequently. If RewriteFrequentPairs is set, the (slow)
 879   // _aload_0 bytecode checks if the next bytecode is either
 880   // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
 881   // rewrites the current bytecode into a pair bytecode; otherwise it
 882   // rewrites the current bytecode into _fast_aload_0 that doesn't do
 883   // the pair check anymore.
 884   //
 885   // Note: If the next bytecode is _getfield, the rewrite must be
 886   //       delayed, otherwise we may miss an opportunity for a pair.
 887   //
 888   // Also rewrite frequent pairs
 889   //   aload_0, aload_1
 890   //   aload_0, iload_1
 891   // These bytecodes with a small amount of code are most profitable
 892   // to rewrite
 893   if (RewriteFrequentPairs) {
 894     Label rewrite, done;
 895     const Register bc = r14;
 896 
 897     // get next bytecode
 898     __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
 899 
 900     // do actual aload_0
 901     aload(0);
 902 
 903     // if _getfield then wait with rewrite
 904     __ cmp(r1, Bytecodes::Bytecodes::_getfield);
 905     __ b(done, Assembler::EQ);
 906 
 907     // if _igetfield then reqrite to _fast_iaccess_0
 908     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 909     __ cmp(r1, Bytecodes::_fast_igetfield);
 910     __ mov(bc, Bytecodes::_fast_iaccess_0);
 911     __ b(rewrite, Assembler::EQ);
 912 
 913     // if _agetfield then reqrite to _fast_aaccess_0
 914     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 915     __ cmp(r1, Bytecodes::_fast_agetfield);
 916     __ mov(bc, Bytecodes::_fast_aaccess_0);
 917     __ b(rewrite, Assembler::EQ);
 918 
 919     // if _fgetfield then reqrite to _fast_faccess_0
 920     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
 921     __ cmp(r1, Bytecodes::_fast_fgetfield);
 922     __ mov(bc, Bytecodes::_fast_faccess_0);
 923     __ b(rewrite, Assembler::EQ);
 924 
 925     // else rewrite to _fast_aload0
 926     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
 927     __ mov(bc, Bytecodes::Bytecodes::_fast_aload_0);
 928 
 929     // rewrite
 930     // bc: new bytecode
 931     __ bind(rewrite);
 932     patch_bytecode(Bytecodes::_aload_0, bc, r1, false);
 933 
 934     __ bind(done);
 935   } else {
 936     aload(0);
 937   }
 938 }
 939 
 940 void TemplateTable::istore()
 941 {
 942   transition(itos, vtos);
 943   locals_index(r1);
 944   __ lea(rscratch1, iaddress(r1));
 945   __ str(r0, Address(rscratch1));
 946 }
 947 
 948 void TemplateTable::lstore()
 949 {
 950   transition(ltos, vtos);
 951   locals_index(r2);
 952   __ strd(r0, r1, laddress(r2, r3, _masm));
 953 }
 954 
 955 void TemplateTable::fstore() {
 956   transition(ftos, vtos);
 957   locals_index(r1);
 958   __ lea(rscratch1, iaddress(r1));
 959   if (hasFPU()) {
 960       __ vstr_f32(d0, Address(rscratch1));
 961   } else {
 962     __ str(r0, Address(rscratch1));
 963   }
 964 }
 965 
 966 void TemplateTable::dstore() {
 967   transition(dtos, vtos);
 968   if (hasFPU()) {
 969     locals_index(r1);
 970     __ vstr_f64(d0, daddress(r1, rscratch1, _masm));
 971   } else {
 972     locals_index(r2);
 973     __ strd(r0, r1, daddress(r2, rscratch1, _masm));
 974   }
 975 }
 976 
 977 void TemplateTable::astore()
 978 {
 979   transition(vtos, vtos);
 980   __ pop_ptr(r0);
 981   __ reg_printf("Astore, storing value %p\n", r0);
 982   locals_index(r1);
 983   __ str(r0, aaddress(r1));
 984 }
 985 
 986 void TemplateTable::wide_istore() {
 987   transition(vtos, vtos);
 988   __ pop_i();
 989   locals_index_wide(r1);
 990   __ lea(rscratch1, iaddress(r1));
 991   __ str(r0, Address(rscratch1));
 992 }
 993 
 994 void TemplateTable::wide_lstore() {
 995   transition(vtos, vtos);
 996   __ pop_l();
 997   locals_index_wide(r2);
 998   __ strd(r0, r1, laddress(r2, r3, _masm));
 999 }
1000 
1001 void TemplateTable::wide_fstore() {
1002   transition(vtos, vtos);
1003   locals_index_wide(r1);
1004   __ lea(rscratch1, faddress(r1, rscratch1, _masm));
1005   if (hasFPU()) {
1006       __ pop_f();
1007     __ vstr_f32(d0, rscratch1);
1008   } else {
1009     __ pop_i();
1010     __ str(r0, Address(rscratch1));
1011   }
1012 }
1013 
1014 void TemplateTable::wide_dstore() {
1015   transition(vtos, vtos);
1016   if (hasFPU()) {
1017     __ pop_d();
1018     locals_index_wide(r1);
1019     __ vstr_f64(d0, daddress(r1, rscratch1, _masm));
1020   } else {
1021     __ pop_l();
1022     locals_index_wide(r2);
1023     __ strd(r0, r1, daddress(r2, rscratch1, _masm));
1024   }
1025 }
1026 
1027 void TemplateTable::wide_astore() {
1028   transition(vtos, vtos);
1029   __ pop_ptr(r0);
1030   locals_index_wide(r1);
1031   __ str(r0, aaddress(r1));
1032 }
1033 
1034 void TemplateTable::iastore() {
1035   transition(itos, vtos);
1036   __ pop_i(r2);
1037   __ pop_ptr(r3);
1038   // r0: value
1039   // r2: index
1040   // r3: array
1041   index_check(r3, r2); // prefer index in r2
1042   __ lea(rscratch1, Address(r3, r2, lsl(2)));
1043   __ str(r0, Address(rscratch1,
1044                      arrayOopDesc::base_offset_in_bytes(T_INT)));
1045 }
1046 
1047 void TemplateTable::lastore() {
1048   transition(ltos, vtos);
1049   __ pop_i(r2);
1050   __ pop_ptr(r3);
1051   // <r0:r1>: value
1052   // r2: index
1053   // r3: array
1054   index_check(r3, r2); // prefer index in r2
1055   __ lea(rscratch1, Address(r3, r2, lsl(3)));
1056   __ lea(rscratch1, Address(rscratch1,
1057                             arrayOopDesc::base_offset_in_bytes(T_LONG)));
1058   __ atomic_strd(r0, r1, rscratch1, r2, r3);
1059 }
1060 
1061 void TemplateTable::fastore() {
1062   transition(ftos, vtos);
1063   __ pop_i(r2);
1064   __ pop_ptr(r3);
1065   // d0/r0: value
1066   // r2:  index
1067   // r3:  array
1068   index_check(r3, r2); // prefer index in r2
1069   __ lea(rscratch1, Address(r3, r2, lsl(2)));
1070   if (hasFPU()) {
1071     __ vstr_f32(d0, Address(rscratch1,
1072                         arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
1073   } else {
1074         __ str(r0, Address(rscratch1,
1075                            arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
1076   }
1077 }
1078 
1079 void TemplateTable::dastore() {
1080   transition(dtos, vtos);
1081   __ pop_i(r2);
1082   __ pop_ptr(r3);
1083   // d0/r0:r1: value
1084   // r2:  index
1085   // r3:  array
1086   index_check(r3, r2); // prefer index in r2
1087   __ lea(rscratch1, Address(r3, r2, lsl(3)));
1088   __ lea(rscratch1, Address(rscratch1,
1089                             arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
1090     if (hasFPU()) {
1091         __ vmov_f64(r0, r1, d0);
1092     }
1093   __ atomic_strd(r0, r1, rscratch1, r2, r3);
1094 }
1095 
1096 void TemplateTable::aastore() {
1097   Label is_null, ok_is_subtype, done;
1098   transition(vtos, vtos);
1099   // stack: ..., array, index, value
1100   __ ldr(r0, at_tos());    // value
1101   __ ldr(r2, at_tos_p1()); // index
1102   __ ldr(r3, at_tos_p2()); // array
1103 
1104   Address element_address(r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
1105 
1106   index_check(r3, r2);
1107 
1108   // do array store check - check for NULL value first
1109   __ cmp(r0, 0);
1110   __ b(is_null, Assembler::EQ);
1111 
1112   // Move subklass into r1
1113   __ load_klass(r1, r0);
1114   // Move superklass into r0
1115   __ load_klass(r0, r3);
1116   __ ldr(r0, Address(r0,
1117                      ObjArrayKlass::element_klass_offset()));
1118   // Compress array + index*oopSize + 12 into a single register.  Frees r2.
1119 
1120   // Generate subtype check.  Blows r2, r14?
1121   // Superklass in r0.  Subklass in r1.
1122   __ gen_subtype_check(r1, ok_is_subtype);
1123 
1124   // Come here on failure
1125   // object is at TOS
1126   __ b(Interpreter::_throw_ArrayStoreException_entry);
1127 
1128   // Come here on success
1129   __ bind(ok_is_subtype);
1130 
1131   // Get the value we will store
1132   __ ldr(r0, at_tos());
1133   // And the clobbered index
1134   __ ldr(r2, at_tos_p1()); // index
1135   __ lea(r1, Address(r3, r2, lsl(2)));
1136   // Now store using the appropriate barrier
1137 
1138   do_oop_store(_masm, element_address, r0, _bs->kind(), true);
1139   __ b(done);
1140 
1141   // Have a NULL in r0, r3=array, r2=index.  Store NULL at ary[idx]
1142   __ bind(is_null);
1143   __ profile_null_seen(r1);
1144 
1145   __ lea(r1, Address(r3, r2, lsl(2)));
1146   // Store a NULL
1147   do_oop_store(_masm, element_address, noreg, _bs->kind(), true);
1148 
1149   // Pop stack arguments
1150   __ bind(done);
1151   __ add(sp, sp, 3 * Interpreter::stackElementSize);
1152 }
1153 
1154 void TemplateTable::bastore()
1155 {
1156   transition(itos, vtos);
1157   __ pop_i(r2);
1158   __ pop_ptr(r3);
1159   // r0: value
1160   // r2: index
1161   // r3: array
1162   index_check(r3, r2); // prefer index in r2
1163 
1164   // Need to check whether array is boolean or byte
1165   // since both types share the bastore bytecode.
1166   __ load_klass(r1, r3);
1167   __ ldr(r1, Address(r1, Klass::layout_helper_offset()));
1168   int diffbit = Klass::layout_helper_boolean_diffbit();
1169   __ tst(r1, diffbit);
1170   __ andr(r0, r0, 1, Assembler::NE);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
1171 
1172   __ lea(rscratch1, Address(r3, r2));
1173   __ strb(r0, Address(rscratch1,
1174                       arrayOopDesc::base_offset_in_bytes(T_BYTE)));
1175 }
1176 
1177 void TemplateTable::castore()
1178 {
1179   transition(itos, vtos);
1180   __ pop_i(r2);
1181   __ pop_ptr(r3);
1182   // r0: value
1183   // r2: index
1184   // r3: array
1185   index_check(r3, r2); // prefer index in r2
1186   __ lea(rscratch1, Address(r3, r2, lsl(1)));
1187   __ strh(r0, Address(rscratch1,
1188                       arrayOopDesc::base_offset_in_bytes(T_CHAR)));
1189 }
1190 
1191 void TemplateTable::sastore()
1192 {
1193   castore();
1194 }
1195 
1196 void TemplateTable::istore(int n)
1197 {
1198   transition(itos, vtos);
1199   __ str(r0, iaddress(n));
1200 }
1201 
1202 void TemplateTable::lstore(int n)
1203 {
1204   transition(ltos, vtos);
1205   __ strd(r0, r1, laddress(n));
1206 }
1207 
1208 void TemplateTable::fstore(int n)
1209 {
1210   transition(ftos, vtos);
1211   if (hasFPU()) {
1212       __ vstr_f32(d0, faddress(n));
1213   } else {
1214     __ str(r0, faddress(n));
1215   }
1216 }
1217 
1218 void TemplateTable::dstore(int n)
1219 {
1220   transition(dtos, vtos);
1221   if (hasFPU()) {
1222       __ vstr_f64(d0, daddress(n));
1223   } else {
1224     __ strd(r0, r1, daddress(n));
1225   }
1226 }
1227 
1228 void TemplateTable::astore(int n)
1229 {
1230   transition(vtos, vtos);
1231   __ pop_ptr(r0);
1232   __ str(r0, iaddress(n));
1233 }
1234 
1235 void TemplateTable::pop()
1236 {
1237   transition(vtos, vtos);
1238   __ add(sp, sp, Interpreter::stackElementSize);
1239 }
1240 
1241 void TemplateTable::pop2()
1242 {
1243   transition(vtos, vtos);
1244   __ add(sp, sp, 2 * Interpreter::stackElementSize);
1245 }
1246 
1247 void TemplateTable::dup()
1248 {
1249   transition(vtos, vtos);
1250   __ ldr(r0, Address(sp, 0));
1251   __ reg_printf("Value duplicated is %p\n", r0);
1252   __ push(r0);
1253   // stack: ..., a, a
1254 }
1255 
1256 void TemplateTable::dup_x1()
1257 {
1258   transition(vtos, vtos);
1259   // stack: ..., a, b
1260   __ ldr(r0, at_tos());  // load b
1261   __ ldr(r2, at_tos_p1());  // load a
1262   __ str(r0, at_tos_p1());  // store b
1263   __ str(r2, at_tos());  // store a
1264   __ push(r0);                  // push b
1265   // stack: ..., b, a, b
1266 }
1267 
1268 void TemplateTable::dup_x2()
1269 {
1270   transition(vtos, vtos);
1271   // stack: ..., a, b, c
1272   __ ldr(r0, at_tos());  // load c
1273   __ ldr(r2, at_tos_p2());  // load a
1274   __ str(r0, at_tos_p2());  // store c in a
1275   __ push(r0);      // push c
1276   // stack: ..., c, b, c, c
1277   __ ldr(r0, at_tos_p2());  // load b
1278   __ str(r2, at_tos_p2());  // store a in b
1279   // stack: ..., c, a, c, c
1280   __ str(r0, at_tos_p1());  // store b in c
1281   // stack: ..., c, a, b, c
1282 }
1283 
1284 void TemplateTable::dup2()
1285 {
1286   transition(vtos, vtos);
1287   // stack: ..., a, b
1288   __ ldr(r0, at_tos_p1());  // load a
1289   __ push(r0);                  // push a
1290   __ ldr(r0, at_tos_p1());  // load b
1291   __ push(r0);                  // push b
1292   // stack: ..., a, b, a, b
1293 }
1294 
1295 void TemplateTable::dup2_x1()
1296 {
1297   transition(vtos, vtos);
1298   // stack: ..., a, b, c
1299   __ ldr(r2, at_tos());  // load c
1300   __ ldr(r0, at_tos_p1());  // load b
1301   __ push(r0);                  // push b
1302   __ push(r2);                  // push c
1303   // stack: ..., a, b, c, b, c
1304   __ str(r2, at_tos_p3());  // store c in b
1305   // stack: ..., a, c, c, b, c
1306   __ ldr(r2, at_tos_p4());  // load a
1307   __ str(r2, at_tos_p2());  // store a in 2nd c
1308   // stack: ..., a, c, a, b, c
1309   __ str(r0, at_tos_p4());  // store b in a
1310   // stack: ..., b, c, a, b, c
1311 }
1312 
1313 void TemplateTable::dup2_x2()
1314 {
1315   transition(vtos, vtos);
1316   // stack: ..., a, b, c, d
1317   __ ldr(r2, at_tos());  // load d
1318   __ ldr(r0, at_tos_p1());  // load c
1319   __ push(r0)            ;      // push c
1320   __ push(r2);                  // push d
1321   // stack: ..., a, b, c, d, c, d
1322   __ ldr(r0, at_tos_p4());  // load b
1323   __ str(r0, at_tos_p2());  // store b in d
1324   __ str(r2, at_tos_p4());  // store d in b
1325   // stack: ..., a, d, c, b, c, d
1326   __ ldr(r2, at_tos_p5());  // load a
1327   __ ldr(r0, at_tos_p3());  // load c
1328   __ str(r2, at_tos_p3());  // store a in c
1329   __ str(r0, at_tos_p5());  // store c in a
1330   // stack: ..., c, d, a, b, c, d
1331 }
1332 
1333 void TemplateTable::swap()
1334 {
1335   transition(vtos, vtos);
1336   // stack: ..., a, b
1337   __ ldr(r2, at_tos_p1());  // load a
1338   __ ldr(r0, at_tos());  // load b
1339   __ str(r2, at_tos());  // store a in b
1340   __ str(r0, at_tos_p1());  // store b in a
1341   // stack: ..., b, a
1342 }
1343 
1344 void TemplateTable::iop2(Operation op)
1345 {
1346   transition(itos, itos);
1347   // r0 <== r1 op r0
1348   __ pop_i(r1);
1349   switch (op) {
1350   case add  : __ add(r0, r1, r0);  break;
1351   case sub  : __ sub(r0, r1, r0);  break;
1352   case mul  : __ mul(r0, r1, r0);  break;
1353   case _and : __ andr(r0, r1, r0); break;
1354   case _or  : __ orr(r0, r1, r0);  break;
1355   case _xor : __ eor(r0, r1, r0);  break;
1356   case shl  :
1357       __ andr(r0, r0, 0x1f);
1358       __ lsl(r0, r1, r0);
1359       break;
1360   case shr  :
1361       __ andr(r0, r0, 0x1f);
1362       __ asr(r0, r1, r0);
1363       break;
1364   case ushr :
1365       __ andr(r0, r0, 0x1f);
1366       __ lsr(r0, r1, r0);
1367       break;
1368   default   : ShouldNotReachHere();
1369   }
1370 }
1371 
1372 void TemplateTable::lop2(Operation op)
1373 {
1374   transition(ltos, ltos);
1375   // <r1:r0> <== <r3:r2> op <r1:r0>
1376   __ pop_l(r2, r3);
1377   switch (op) {
1378   case add  : __ adds(r0, r2, r0); __ adc(r1, r3, r1);  break;
1379   case sub  : __ subs(r0, r2, r0); __ sbc(r1, r3, r1);  break;
1380   case mul  : __ mult_long(r0, r2, r0);                 break;
1381   case _and : __ andr(r0, r2, r0); __ andr(r1, r3, r1); break;
1382   case _or  : __ orr(r0, r2, r0);  __ orr(r1, r3, r1);  break;
1383   case _xor : __ eor(r0, r2, r0);  __ eor(r1, r3, r1);  break;
1384   default   : ShouldNotReachHere();
1385   }
1386 }
1387 
1388 void TemplateTable::idiv()
1389 {
1390   transition(itos, itos);
1391   // explicitly check for div0
1392   Label no_div0;
1393   __ cmp(r0, 0);
1394   __ b(no_div0, Assembler::NE);
1395   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1396   __ b(rscratch1);
1397   __ bind(no_div0);
1398   __ pop_i(r1);
1399   // r0 <== r1 idiv r0
1400   __ divide(r0, r1, r0, 32, false);
1401 }
1402 
1403 void TemplateTable::irem()
1404 {
1405   transition(itos, itos);
1406   // explicitly check for div0
1407   Label no_div0;
1408   __ cmp(r0, 0);
1409   __ b(no_div0, Assembler::NE);
1410   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1411   __ b(rscratch1);
1412   __ bind(no_div0);
1413   __ pop_i(r1);
1414   // r0 <== r1 irem r0
1415   __ divide(r0, r1, r0, 32, true);
1416 }
1417 
1418 void TemplateTable::lmul()
1419 {
1420   transition(ltos, ltos);
1421   __ pop_l(r2, r3);
1422   __ mult_long(r0, r0, r2);
1423 }
1424 
1425 void TemplateTable::ldiv()
1426 {
1427   transition(ltos, ltos);
1428   // explicitly check for div0
1429   __ cmp(r0, 0);
1430   __ cmp(r1, 0, Assembler::EQ);
1431   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry, Assembler::EQ);
1432   __ b(rscratch1, Assembler::EQ);
1433 
1434   __ pop_l(r2, r3);
1435   // r0 <== r1 ldiv r0
1436   __ divide(r0, r2, r0, 64, false);
1437 }
1438 
1439 void TemplateTable::lrem()
1440 {
1441   transition(ltos, ltos);
1442   // explicitly check for div0
1443   __ cmp(r0, 0);
1444   __ cmp(r1, 0, Assembler::EQ);
1445   __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry, Assembler::EQ);
1446   __ b(rscratch1, Assembler::EQ);
1447 
1448   __ pop_l(r2, r3);
1449   // r0 <== r1 lrem r0
1450   __ divide(r0, r2, r0, 64, true);
1451 }
1452 
1453 void TemplateTable::lshl() {
1454     transition(itos, ltos);
1455     // shift count is in r0 - take shift from bottom six bits only
1456     __ andr(r0, r0, 0x3f);
1457     __ pop_l(r2, r3);
1458     const int word_bits = 8 * wordSize;
1459 
1460     __ sub(r1, r0, word_bits);
1461     __ lsl(r3, r3, r0);
1462     __ orr(r3, r3, r2, lsl(r1));
1463     __ rsb(r1, r0, word_bits);
1464     __ orr(r1, r3, r2, lsr(r1));
1465     __ lsl(r0, r2, r0);
1466 }
1467 
1468 void TemplateTable::lshr() {
1469     transition(itos, ltos);
1470     // shift count is in r0 - take shift from bottom six bits only
1471     __ andr(rscratch1, r0, 0x3f);
1472     __ pop_l(r2, r3);
1473     const int word_bits = 8 * wordSize;
1474 
1475     __ lsr(r2, r2, rscratch1);
1476     __ rsb(r1, rscratch1, word_bits);
1477     __ orr(r0, r2, r3, lsl(r1));
1478     __ asr(r1, r3, rscratch1);
1479     __ subs(rscratch1, rscratch1, word_bits);
1480     __ orr(r0, r2, r3, asr(rscratch1), Assembler::GT);
1481 }
1482 
1483 void TemplateTable::lushr() {
1484     transition(itos, ltos);
1485     // shift count is in r0 - take shift from bottom six bits only
1486     __ andr(r0, r0, 0x3f);
1487     __ pop_l(r2, r3);
1488     const int word_bits = 8 * wordSize;
1489 
1490     __ lsr(r2, r2, r0);
1491     __ rsb(r1, r0, word_bits);
1492     __ orr(r2, r2, r3, lsl(r1));
1493     __ lsr(r1, r3, r0);
1494     __ sub(r0, r0, word_bits);
1495     __ orr(r0, r2, r3, lsr(r0));
1496 }
1497 
1498 void TemplateTable::fop2(Operation op)
1499 {
1500   transition(ftos, ftos);
1501   if(hasFPU()) {
1502     switch (op) {
1503     case add:
1504       __ pop_f(d1);
1505       __ vadd_f32(d0, d1, d0);
1506       break;
1507     case sub:
1508       __ pop_f(d1);
1509       __ vsub_f32(d0, d1, d0);
1510       break;
1511     case mul:
1512       __ pop_f(d1);
1513       __ vmul_f32(d0, d1, d0);
1514       break;
1515     case div:
1516       __ pop_f(d1);
1517       __ vdiv_f32(d0, d1, d0);
1518       break;
1519     case rem:
1520       __ vmov_f32(f1, f0);
1521       __ pop_f(f0);
1522       #ifndef HARD_FLOAT_CC
1523       __ vmov_f32(r0, f0);
1524       __ vmov_f32(r1, f1);
1525       #endif
1526       __ mov(rscratch1, (address)fmodf);
1527       __ bl(rscratch1);
1528       #ifndef HARD_FLOAT_CC
1529       __ vmov_f32(f0, r0);
1530       #endif
1531       break;
1532     default:
1533       ShouldNotReachHere();
1534       break;
1535     }
1536   } else {
1537 #ifdef __SOFTFP__
1538     __ mov(r1, r0);
1539     __ pop_i(r0);
1540     switch (op) {
1541     case add:
1542       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::fadd), 0);
1543       break;
1544     case sub:
1545       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::fsub), 0);
1546       break;
1547     case mul:
1548       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::fmul), 0);
1549       break;
1550     case div:
1551       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::fdiv), 0);
1552       break;
1553     case rem:
1554       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::frem), 0);
1555       break;
1556     default:
1557       ShouldNotReachHere();
1558       break;
1559     }
1560   #else
1561       // expected -mfloat-abi=soft
1562       ShouldNotReachHere();
1563 #endif
1564 }
1565 }
1566 
1567 void TemplateTable::dop2(Operation op)
1568 {
1569   transition(dtos, dtos);
1570   if (hasFPU()) {
1571     switch (op) {
1572     case add:
1573       __ pop_d(d1);
1574       __ vadd_f64(d0, d1, d0);
1575       break;
1576     case sub:
1577       __ pop_d(d1);
1578       __ vsub_f64(d0, d1, d0);
1579       break;
1580     case mul:
1581       __ pop_d(d1);
1582       __ vmul_f64(d0, d1, d0);
1583       break;
1584     case div:
1585       __ pop_d(d1);
1586       __ vdiv_f64(d0, d1, d0);
1587       break;
1588     case rem:
1589       __ vmov_f64(d1, d0);
1590       __ pop_d(d0);
1591       #ifndef HARD_FLOAT_CC
1592       __ vmov_f64(r0, r1, d0);
1593       __ vmov_f64(r2, r3, d1);
1594       #endif
1595       __ mov(rscratch1, (address)(double (*)(double, double))fmod);
1596       __ bl(rscratch1);
1597       #ifndef HARD_FLOAT_CC
1598       __ vmov_f64(d0, r0, r1);
1599       #endif
1600       break;
1601     default:
1602       ShouldNotReachHere();
1603       break;
1604     }
1605   } else {
1606 #ifdef __SOFTFP__
1607     __ push_l(r0, r1);
1608     __ pop_l(r2,r3);
1609     __ pop_l(r0,r1);
1610     switch (op) {
1611     case add:
1612       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::dadd), 0);
1613       break;
1614     case sub:
1615       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::dsub), 0);
1616       break;
1617     case mul:
1618       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::dmul), 0);
1619       break;
1620     case div:
1621       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::ddiv), 0);
1622       break;
1623     case rem:
1624       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::drem), 0);
1625       break;
1626     default:
1627       ShouldNotReachHere();
1628       break;
1629     }
1630 #else
1631       // expected -mfloat-abi=soft
1632       ShouldNotReachHere();
1633 #endif
1634   }
1635 }
1636 
1637 void TemplateTable::ineg()
1638 {
1639   transition(itos, itos);
1640   __ neg(r0, r0);
1641 
1642 }
1643 
1644 void TemplateTable::lneg()
1645 {
1646   transition(ltos, ltos);
1647   __ rsbs(r0, r0, 0);
1648   __  rsc(r1, r1, 0);
1649 }
1650 
1651 void TemplateTable::fneg()
1652 {
1653   transition(ftos, ftos);
1654   if(hasFPU()) {
1655       __ vneg_f32(d0, d0);
1656   } else {
1657 #ifdef __SOFTFP__
1658     __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::fneg), 0);
1659 #else
1660       // expected -mfloat-abi=soft
1661       ShouldNotReachHere();
1662 #endif
1663   }
1664 }
1665 
1666 void TemplateTable::dneg()
1667 {
1668   transition(dtos, dtos);
1669   if(hasFPU()) {
1670       __ vneg_f64(d0, d0);
1671   } else {
1672 #ifdef __SOFTFP__
1673       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::dneg), 0);
1674 #else
1675       // expected -mfloat-abi=soft
1676       ShouldNotReachHere();
1677 #endif
1678   }
1679 }
1680 
1681 void TemplateTable::iinc()
1682 {
1683   transition(vtos, vtos);
1684   __ load_signed_byte(r1, at_bcp(2)); // get constant
1685   locals_index(r2);
1686   __ ldr(r0, iaddress(r2));
1687   __ add(r0, r0, r1);
1688   __ str(r0, iaddress(r2));
1689 }
1690 
1691 void TemplateTable::wide_iinc()
1692 {
1693   transition(vtos, vtos);
1694   __ ldr(r1, at_bcp(2)); // get constant and index
1695   __ rev16(r1, r1);
1696   __ uxth(r2, r1);
1697   __ neg(r2, r2);
1698   __ sxth(r1, r1, ror(16));
1699   __ ldr(r0, iaddress(r2));
1700   __ add(r0, r0, r1);
1701   __ str(r0, iaddress(r2));
1702 }
1703 
1704 void TemplateTable::convert()
1705 {
1706   // Checking
1707 #ifdef ASSERT
1708   {
1709     TosState tos_in  = ilgl;
1710     TosState tos_out = ilgl;
1711     switch (bytecode()) {
1712     case Bytecodes::_i2l: // fall through
1713     case Bytecodes::_i2f: // fall through
1714     case Bytecodes::_i2d: // fall through
1715     case Bytecodes::_i2b: // fall through
1716     case Bytecodes::_i2c: // fall through
1717     case Bytecodes::_i2s: tos_in = itos; break;
1718     case Bytecodes::_l2i: // fall through
1719     case Bytecodes::_l2f: // fall through
1720     case Bytecodes::_l2d: tos_in = ltos; break;
1721     case Bytecodes::_f2i: // fall through
1722     case Bytecodes::_f2l: // fall through
1723     case Bytecodes::_f2d: tos_in = ftos; break;
1724     case Bytecodes::_d2i: // fall through
1725     case Bytecodes::_d2l: // fall through
1726     case Bytecodes::_d2f: tos_in = dtos; break;
1727     default             : ShouldNotReachHere();
1728     }
1729     switch (bytecode()) {
1730     case Bytecodes::_l2i: // fall through
1731     case Bytecodes::_f2i: // fall through
1732     case Bytecodes::_d2i: // fall through
1733     case Bytecodes::_i2b: // fall through
1734     case Bytecodes::_i2c: // fall through
1735     case Bytecodes::_i2s: tos_out = itos; break;
1736     case Bytecodes::_i2l: // fall through
1737     case Bytecodes::_f2l: // fall through
1738     case Bytecodes::_d2l: tos_out = ltos; break;
1739     case Bytecodes::_i2f: // fall through
1740     case Bytecodes::_l2f: // fall through
1741     case Bytecodes::_d2f: tos_out = ftos; break;
1742     case Bytecodes::_i2d: // fall through
1743     case Bytecodes::_l2d: // fall through
1744     case Bytecodes::_f2d: tos_out = dtos; break;
1745     default             : ShouldNotReachHere();
1746     }
1747     transition(tos_in, tos_out);
1748   }
1749 #endif // ASSERT
1750   // static const int64_t is_nan = 0x8000000000000000L;
1751   //TODO fix this and remove _ sxtw and _ uxtw as don't exist in arm32
1752   // need to figure out about handling doubles and longs as they won't
1753   // fit into a single register in arm32
1754   // Conversion
1755   switch (bytecode()) {
1756   case Bytecodes::_i2l:
1757     // __ sxtw(r0, r0);
1758     __ reg_printf("Convert i2l (before) 0x00000000%08x\n", r0);
1759     __ asr(r1, r0, 31);
1760     __ reg_printf("Convert i2l (after) 0x%08x%08x\n", r1, r0);
1761     break;
1762   case Bytecodes::_i2f:
1763     if(hasFPU()) {
1764       __ vmov_f32(d0, r0);
1765       __ vcvt_f32_s32(d0, d0);
1766     } else {
1767 #ifdef __SOFTFP__
1768       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::i2f), 0);
1769 #else
1770       // expected -mfloat-abi=soft
1771       ShouldNotReachHere();
1772 #endif
1773     }
1774     break;
1775   case Bytecodes::_i2d:
1776     if(hasFPU()) {
1777       //__ scvtfwd(d0, r0);
1778       __ vmov_f32(d0, r0);
1779       __ vcvt_f64_s32(d0, d0);
1780     } else {
1781 #ifdef __SOFTFP__
1782         // ro -> <r1:r0>
1783       __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::i2d), 0);
1784 #else
1785       // expected -mfloat-abi=soft
1786       ShouldNotReachHere();
1787 #endif
1788     }
1789     break;
1790   case Bytecodes::_i2b:
1791     __ sxtb(r0, r0);
1792     break;
1793   case Bytecodes::_i2c:
1794     __ uxth(r0, r0);
1795     break;
1796   case Bytecodes::_i2s:
1797     __ sxth(r0, r0);
1798     break;
1799   case Bytecodes::_l2i:
1800     //__ uxtw(r0, r0);
1801     break;
1802   case Bytecodes::_l2f:
1803     // <r1:r0> -> d0
1804     // or <r1:r0> -> r0 for softfp
1805     __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::l2f), 0);
1806 #ifndef HARD_FLOAT_CC
1807     if(hasFPU()) {
1808         __ vmov_f32(d0, r0);
1809     }
1810 #endif
1811     break;
1812   case Bytecodes::_l2d:
1813     // <r1:r0> -> d0
1814     // or <r1:r0> -> <r1:r0> for softfp
1815     __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::l2d), 0);
1816 #ifndef HARD_FLOAT_CC
1817     if(hasFPU()) {
1818         __ vmov_f64(d0, r0, r1);
1819     }
1820 #endif
1821     break;
1822   case Bytecodes::_f2i:
1823   {
1824       if(hasFPU()) {
1825         __ vcvt_s32_f32(d0, d0);
1826         __ vmov_f32(r0, d0);
1827       } else {
1828         __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::f2i), 0);
1829       }
1830   }
1831     break;
1832   case Bytecodes::_f2l:
1833   {
1834 #if !defined(HARD_FLOAT_CC)
1835     //float already in d0 long goes to <r1:r0>
1836     if(hasFPU()) {
1837         //Need to move float in d0 to r0
1838         __ vmov_f32(r0, d0);
1839     }
1840 #endif //!defined(HARD_FLOAT_CC)
1841     __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::f2l), 0);
1842   }
1843     break;
1844   case Bytecodes::_f2d:
1845     if(hasFPU()) {
1846         __ vcvt_f64_f32(d0, d0);
1847     } else {
1848 #ifdef __SOFTFP__
1849         __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::f2d), 0);
1850 #else
1851       // expected -mfloat-abi=soft
1852       ShouldNotReachHere();
1853 #endif
1854     }
1855     break;
1856   case Bytecodes::_d2i:
1857   {
1858     if(hasFPU()) {
1859         __ vcvt_s32_f64(d0, d0);
1860         __ vmov_f32(r0, d0);
1861     } else {
1862 #ifdef __SOFTFP__
1863         __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::d2i), 0);
1864 #else
1865         // expected -mfloat-abi=soft
1866         ShouldNotReachHere();
1867 #endif
1868     }
1869   }
1870     break;
1871   case Bytecodes::_d2l:
1872   {
1873     // d0 -> <r1:r0>
1874 #if !defined(HARD_FLOAT_CC)
1875     if(hasFPU()) {
1876         //Need to move float in d0 to r0
1877         __ vmov_f64(r0, r1, d0);
1878     }
1879 #endif //!defined(HARD_FLOAT_CC)
1880     __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::d2l), 0);
1881   }
1882     break;
1883   case Bytecodes::_d2f:
1884     if(hasFPU()) {
1885         __ vcvt_f32_f64(d0, d0);
1886     } else {
1887 #ifdef __SOFTFP__
1888         __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::d2f), 0);
1889 #else
1890       // expected -mfloat-abi=soft
1891       ShouldNotReachHere();
1892 #endif
1893     }
1894     break;
1895   default:
1896     ShouldNotReachHere();
1897   }
1898 }
1899 
1900 void TemplateTable::lcmp()
1901 {
1902   transition(ltos, itos);
1903   __ pop_l(r2, r3);
1904   // <r1:r0> == <r3:r2> : 0
1905   // <r1:r0> < <r3:r2> : 1
1906   // <r1:r0> > <r3:r2> : -1
1907   __ reg_printf("Long comparing 0x%08x%08x\n", r1, r0);
1908   __ reg_printf("           and 0x%08x%08x\n", r3, r2);
1909   //cmp high
1910   Label lower, end;
1911   __ cmp(r3, r1);
1912   __ b(lower, Assembler::EQ);
1913   __ mov(r0, 1);
1914   __ sub(r0, r0, 2, Assembler::LT);
1915   __ b(end);
1916 
1917   __ bind(lower);
1918   __ subs(r0, r2, r0);
1919   __ mov(r0, 1, Assembler::NE);
1920   __ sub(r0, r0, 2, Assembler::LO); // Place -1
1921   __ bind(end);
1922 
1923   __ reg_printf("Result of comparison is %d\n", r0);
1924 }
1925 
1926 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1927 {
1928     if(hasFPU()) {
1929         if (is_float) {
1930          __ pop_f(d1);
1931          __ vcmp_f32(d1, d0);
1932        } else {
1933          __ pop_d(d1);
1934          /*__ vmov_f64(r0, r1, d0);
1935          __ vmov_f64(r2, r3, d1);
1936          __ reg_printf("Doing comparison cmp( 0x%08x%08x,\n", r3, r2);
1937          __ reg_printf("                      0x%08x%08x)\n", r1, r0);*/
1938          __ vcmp_f64(d1, d0);
1939        }
1940        __ vmrs(rscratch1);
1941        __ andr(rscratch1, rscratch1, Assembler::FP_MASK);
1942        __ reg_printf("Masked comparison result is %08x\n", rscratch1);
1943 
1944        if (unordered_result < 0) {
1945          // we want -1 for unordered or less than, 0 for equal and 1 for
1946          // greater than.
1947          __ mov(r0, -1);
1948          __ cmp(rscratch1, Assembler::FP_EQ);
1949          __ mov(r0, 0, Assembler::EQ);
1950          __ cmp(rscratch1, Assembler::FP_GT);
1951          __ mov(r0, 1, Assembler::EQ);
1952          __ reg_printf("un_res < 0, comparison result is %d\n", r0);
1953        } else {
1954          // we want -1 for less than, 0 for equal and 1 for unordered or
1955          // greater than.
1956          __ mov(r0, 1);
1957          __ cmp(rscratch1, Assembler::FP_LT);
1958          __ sub(r0, r0, 2, Assembler::EQ); //Load -1 - but one less instruction
1959          __ cmp(rscratch1, Assembler::FP_EQ);
1960          __ mov(r0, 0, Assembler::EQ);
1961          __ reg_printf("un_res >= 0, comparison result is %d\n", r0);
1962        }
1963     } else { // hasFPU
1964 #ifdef __SOFTFP__
1965         if (is_float) {
1966             __ mov(r1, r0);
1967             __ pop_i(r0);
1968             if (unordered_result < 0) {
1969                 __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::fcmpl), 0);
1970             } else {
1971                 __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::fcmpg), 0);
1972             }
1973         } else {
1974             __ mov(r2, r0);
1975             __ mov(r3, r1);
1976             __ pop_l(r0);
1977             if (unordered_result < 0) {
1978                 __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::dcmpl), 0);
1979             } else {
1980                 __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::dcmpg), 0);
1981             }
1982         }
1983 #else
1984         // expected -mfloat-abi=soft
1985         ShouldNotReachHere();
1986 #endif
1987   }
1988 }
1989 
1990 void TemplateTable::branch(bool is_jsr, bool is_wide)
1991 {
1992   // We might be moving to a safepoint.  The thread which calls
1993   // Interpreter::notice_safepoints() will effectively flush its cache
1994   // when it makes a system call, but we need to do something to
1995   // ensure that we see the changed dispatch table.
1996   __ membar(MacroAssembler::LoadLoad);
1997 
1998   __ profile_taken_branch(r0, r1);
1999   const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
2000                              InvocationCounter::counter_offset();
2001   const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
2002                               InvocationCounter::counter_offset();
2003 
2004   // load branch displacement
2005   if (!is_wide) {
2006     __ ldrh(r2, at_bcp(1));
2007     __ rev16(r2, r2);
2008     // sign extend the 16 bit value in r2
2009     __ sxth(r2, r2);
2010   } else {
2011     __ ldr(r2, at_bcp(1));
2012     __ rev(r2, r2);
2013   }
2014 
2015   // Handle all the JSR stuff here, then exit.
2016   // It's much shorter and cleaner than intermingling with the non-JSR
2017   // normal-branch stuff occurring below.
2018 
2019   if (is_jsr) {
2020     // Pre-load the next target bytecode into rscratch1
2021     __ load_unsigned_byte(rscratch1, Address(rbcp, r2));
2022     // compute return address as bci
2023     __ ldr(rscratch2, Address(rmethod, Method::const_offset()));
2024     __ add(rscratch2, rscratch2,
2025            in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
2026     __ sub(r1, rbcp, rscratch2);
2027     __ push_i(r1);
2028     // Adjust the bcp by the 16-bit displacement in r2
2029     __ add(rbcp, rbcp, r2);
2030     __ dispatch_only(vtos);
2031     return;
2032   }
2033 
2034   // Normal (non-jsr) branch handling
2035 
2036   // Adjust the bcp by the displacement in r2
2037   __ add(rbcp, rbcp, r2);
2038 
2039   assert(UseLoopCounter || !UseOnStackReplacement,
2040          "on-stack-replacement requires loop counters");
2041   Label backedge_counter_overflow;
2042   Label profile_method;
2043   Label dispatch;
2044   if (UseLoopCounter) {
2045     // increment backedge counter for backward branches
2046     // r0: MDO
2047     // w1: MDO bumped taken-count
2048     // r2: target offset
2049     __ cmp(r2, 0);
2050     __ b(dispatch, Assembler::GT); // count only if backward branch
2051 
2052     // ECN: FIXME: This code smells
2053     // check if MethodCounters exists
2054     Label has_counters;
2055     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
2056     __ cbnz(rscratch1, has_counters);
2057     __ push(r0);
2058     __ push(r1);
2059     __ push(r2);
2060     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2061             InterpreterRuntime::build_method_counters), rmethod);
2062     __ pop(r2);
2063     __ pop(r1);
2064     __ pop(r0);
2065     __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
2066     __ cbz(rscratch1, dispatch); // No MethodCounters allocated, OutOfMemory
2067     __ bind(has_counters);
2068 
2069     if (TieredCompilation) {
2070       Label no_mdo;
2071       int increment = InvocationCounter::count_increment;
2072       int mask = ((1 << Tier0BackedgeNotifyFreqLog) - 1) << InvocationCounter::count_shift;
2073       if (ProfileInterpreter) {
2074         // Are we profiling?
2075         __ ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
2076         __ cbz(r1, no_mdo);
2077         // Increment the MDO backedge counter
2078         const Address mdo_backedge_counter(r1, in_bytes(MethodData::backedge_counter_offset()) +
2079                                            in_bytes(InvocationCounter::counter_offset()));
2080         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
2081                                    r0, rscratch2, false, Assembler::EQ, &backedge_counter_overflow);
2082         __ b(dispatch);
2083       }
2084       __ bind(no_mdo);
2085       // Increment backedge counter in MethodCounters*
2086       __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
2087       __ increment_mask_and_jump(Address(rscratch1, be_offset), increment, mask,
2088                                  r0, rscratch2, false, Assembler::EQ, &backedge_counter_overflow);
2089     } else {
2090       // increment counter
2091       __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
2092       __ ldr(r0, Address(rscratch2, be_offset));        // load backedge counter
2093       __ add(rscratch1, r0, InvocationCounter::count_increment); // increment counter
2094       __ str(rscratch1, Address(rscratch2, be_offset));        // store counter
2095 
2096       __ ldr(r0, Address(rscratch2, inv_offset));    // load invocation counter
2097       __ mov(rscratch1, (unsigned)InvocationCounter::count_mask_value);
2098       __ andr(r0, r0, rscratch1); // and the status bits
2099       __ ldr(rscratch1, Address(rscratch2, be_offset));        // load backedge counter
2100       __ add(r0, r0, rscratch1);        // add both counters
2101 
2102       if (ProfileInterpreter) {
2103         // Test to see if we should create a method data oop
2104         __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterProfileLimit));
2105         __ ldr(rscratch1, rscratch1);
2106         __ cmp(r0, rscratch1);
2107         __ b(dispatch, Assembler::LT);
2108 
2109         // if no method data exists, go to profile method
2110         __ test_method_data_pointer(r0, profile_method);
2111 
2112         if (UseOnStackReplacement) {
2113           // check for overflow against w1 which is the MDO taken count
2114           __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
2115           __ ldr(rscratch1, rscratch1);
2116           __ cmp(r1, rscratch1);
2117           __ b(dispatch, Assembler::LO); // Intel == Assembler::below
2118 
2119           // When ProfileInterpreter is on, the backedge_count comes
2120           // from the MethodData*, which value does not get reset on
2121           // the call to frequency_counter_overflow().  To avoid
2122           // excessive calls to the overflow routine while the method is
2123           // being compiled, add a second test to make sure the overflow
2124           // function is called only once every overflow_frequency.
2125           const int overflow_frequency = 1024;
2126           const int of_mask_lsb = exact_log2(overflow_frequency);
2127           __ bfc(r1, of_mask_lsb, 32 - of_mask_lsb);
2128           __ cmp(r1, 0);
2129           __ b(backedge_counter_overflow, Assembler::EQ);
2130 
2131         }
2132       } else {
2133         if (UseOnStackReplacement) {
2134           // check for overflow against w0, which is the sum of the
2135           // counters
2136           __ lea(rscratch1, ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
2137           __ ldr(rscratch1, rscratch1);
2138           __ cmp(r0, rscratch1);
2139           __ b(backedge_counter_overflow, Assembler::HS); // Intel == Assembler::aboveEqual
2140         }
2141       }
2142     }
2143   }
2144   __ bind(dispatch);
2145 
2146   // Pre-load the next target bytecode into rscratch1
2147   __ load_unsigned_byte(rscratch1, Address(rbcp, 0));
2148 
2149   // continue with the bytecode @ target
2150   // rscratch1: target bytecode
2151   // rbcp: target bcp
2152   __ dispatch_only(vtos);
2153 
2154   if (UseLoopCounter) {
2155     if (ProfileInterpreter) {
2156       // Out-of-line code to allocate method data oop.
2157       __ bind(profile_method);
2158       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
2159       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
2160       __ set_method_data_pointer_for_bcp();
2161       __ b(dispatch);
2162     }
2163 
2164     if (TieredCompilation || UseOnStackReplacement) {
2165       // invocation counter overflow
2166       __ bind(backedge_counter_overflow);
2167       __ neg(r2, r2);
2168       __ add(r2, r2, rbcp);     // branch bcp
2169       // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
2170       __ call_VM(noreg,
2171                  CAST_FROM_FN_PTR(address,
2172                                   InterpreterRuntime::frequency_counter_overflow),
2173                  r2);
2174       if (!UseOnStackReplacement)
2175         __ b(dispatch);
2176     }
2177 
2178     if (UseOnStackReplacement) {
2179       __ load_unsigned_byte(r1, Address(rbcp, 0));  // restore target bytecode
2180 
2181       // r0: osr nmethod (osr ok) or NULL (osr not possible)
2182       // r1: target bytecode
2183       // r2: scratch
2184       __ cbz(r0, dispatch);     // test result -- no osr if null
2185       // nmethod may have been invalidated (VM may block upon call_VM return)
2186       __ ldr(r2, Address(r0, nmethod::entry_bci_offset()));
2187       // InvalidOSREntryBci == -2 which overflows cmpw as unsigned
2188       // use cmnw against -InvalidOSREntryBci which does the same thing
2189       __ cmn(r2, -InvalidOSREntryBci);
2190       __ b(dispatch, Assembler::EQ);
2191 
2192       // We have the address of an on stack replacement routine in r0
2193       // We need to prepare to execute the OSR method. First we must
2194       // migrate the locals and monitors off of the stack.
2195 
2196       __ mov(r4, r0);               // save the nmethod
2197 
2198       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
2199 
2200       // r0 is OSR buffer, move it to expected parameter location
2201       __ mov(j_rarg0, r0);
2202 
2203       // remove activation
2204       // get sender sp
2205       __ ldr(rscratch1,
2206           Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
2207       // remove frame anchor
2208       __ leave();
2209       __ mov(sp, rscratch1);
2210       // Ensure compiled code always sees stack at proper alignment
2211       __ align_stack();
2212 
2213       // and begin the OSR nmethod
2214       __ ldr(rscratch1, Address(r4, nmethod::osr_entry_point_offset()));
2215       __ b(rscratch1);
2216     }
2217   }
2218 }
2219 
2220 
2221 void TemplateTable::if_0cmp(Condition cc)
2222 {
2223   transition(itos, vtos);
2224   // assume branch is more often taken than not (loops use backward branches)
2225   Label not_taken;
2226   /*if (cc == equal) {
2227     __ cmp(r0, 0);
2228     __ b(not_taken, Assembler::NE);
2229   } else if (cc == not_equal) {
2230     __ cmp(r0, 0);
2231     __ b(not_taken, Assembler::EQ);
2232   } else {
2233     __ ands(rscratch1, r0, r0);
2234     __ b(not_taken, j_not(cc));
2235   }*/
2236   __ cmp(r0, 0);
2237   __ b(not_taken, j_not(cc));
2238 
2239   branch(false, false);
2240   __ bind(not_taken);
2241   __ profile_not_taken_branch(r0);
2242 }
2243 
2244 void TemplateTable::if_icmp(Condition cc)
2245 {
2246   transition(itos, vtos);
2247   // assume branch is more often taken than not (loops use backward branches)
2248   Label not_taken;
2249   __ pop_i(r1);
2250   __ reg_printf("Comparing TOS = %p, and SOS = %p\n", r0, r1);
2251   __ cmp(r1, r0);
2252   __ b(not_taken, j_not(cc));
2253   branch(false, false);
2254   __ bind(not_taken);
2255   __ profile_not_taken_branch(r0);
2256 }
2257 
2258 void TemplateTable::if_nullcmp(Condition cc)
2259 {
2260   transition(atos, vtos);
2261   // assume branch is more often taken than not (loops use backward branches)
2262   Label not_taken;
2263   if (cc == equal)
2264     __ cbnz(r0, not_taken);
2265   else
2266     __ cbz(r0, not_taken);
2267   branch(false, false);
2268   __ bind(not_taken);
2269   __ profile_not_taken_branch(r0);
2270 }
2271 
2272 void TemplateTable::if_acmp(Condition cc)
2273 {
2274   transition(atos, vtos);
2275   // assume branch is more often taken than not (loops use backward branches)
2276   Label not_taken;
2277   __ pop_ptr(r1);
2278   __ cmp(r1, r0);
2279   __ b(not_taken, j_not(cc));
2280   branch(false, false);
2281   __ bind(not_taken);
2282   __ profile_not_taken_branch(r0);
2283 }
2284 
2285 void TemplateTable::ret() {
2286   transition(vtos, vtos);
2287   // We might be moving to a safepoint.  The thread which calls
2288   // Interpreter::notice_safepoints() will effectively flush its cache
2289   // when it makes a system call, but we need to do something to
2290   // ensure that we see the changed dispatch table.
2291   __ membar(MacroAssembler::LoadLoad);
2292 
2293   locals_index(r1);
2294   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2295   __ profile_ret(r1, r2);
2296   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2297   __ lea(rbcp, Address(rbcp, r1));
2298   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2299   __ dispatch_next(vtos);
2300 }
2301 
2302 void TemplateTable::wide_ret() {
2303   transition(vtos, vtos);
2304   locals_index_wide(r1);
2305   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2306   __ profile_ret(r1, r2);
2307   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2308   __ lea(rbcp, Address(rbcp, r1));
2309   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2310   __ dispatch_next(vtos);
2311 }
2312 
2313 void TemplateTable::tableswitch() {
2314   Label default_case, continue_execution;
2315   transition(itos, vtos);
2316   // align rbcp
2317   __ lea(r1, at_bcp(BytesPerInt));
2318   __ bic(r1, r1, BytesPerInt - 1);
2319   // load lo & hi
2320   __ ldr(r2, Address(r1, BytesPerInt));
2321   __ ldr(r3, Address(r1, 2 * BytesPerInt));
2322   __ rev(r2, r2);
2323   __ rev(r3, r3);
2324   // check against lo & hi
2325   __ cmp(r0, r2);
2326   __ b(default_case, Assembler::LT);
2327   __ cmp(r0, r3);
2328   __ b(default_case, Assembler::GT);
2329   // lookup dispatch offset
2330   __ sub(r0, r0, r2);
2331   __ lea(r3, Address(r1, r0, lsl(2)));
2332   __ ldr(r3, Address(r3, 3 * BytesPerInt));
2333   __ profile_switch_case(r0, r1, r2);
2334   // continue execution
2335   __ bind(continue_execution);
2336   __ rev(r3, r3);
2337   __ load_unsigned_byte(rscratch1, Address(rbcp, r3));
2338   __ add(rbcp, rbcp, r3);
2339   __ dispatch_only(vtos);
2340   // handle default
2341   __ bind(default_case);
2342   __ profile_switch_default(r0);
2343   __ ldr(r3, Address(r1, 0));
2344   __ b(continue_execution);
2345 }
2346 
2347 void TemplateTable::lookupswitch() {
2348   transition(itos, itos);
2349   __ stop("lookupswitch bytecode should have been rewritten");
2350 }
2351 
2352 void TemplateTable::fast_linearswitch() {
2353   transition(itos, vtos);
2354   Label loop_entry, loop, found, continue_execution;
2355 
2356   __ reg_printf("Linearswitching to value %d\n", r0);
2357 
2358   // bswap r0 so we can avoid bswapping the table entries
2359   __ rev(r0, r0);
2360   // align rbcp
2361   __ lea(r14, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2362                                     // this instruction (change offsets
2363                                     // below)
2364   __ bic(r14, r14, BytesPerInt - 1);
2365   // set counter
2366   __ ldr(r1, Address(r14, BytesPerInt));
2367   __ rev(r1, r1);
2368   __ b(loop_entry);
2369   // table search
2370   __ bind(loop);
2371   __ lea(rscratch1, Address(r14, r1, lsl(3)));
2372   __ ldr(rscratch1, Address(rscratch1, 2 * BytesPerInt));
2373   __ cmp(r0, rscratch1);
2374   __ b(found, Assembler::EQ);
2375   __ bind(loop_entry);
2376   __ subs(r1, r1, 1);
2377   __ b(loop, Assembler::PL);
2378   // default case
2379   __ profile_switch_default(r0);
2380   __ ldr(r3, Address(r14, 0));
2381   __ b(continue_execution);
2382   // entry found -> get offset
2383   __ bind(found);
2384   __ lea(rscratch1, Address(r14, r1, lsl(3)));
2385   __ ldr(r3, Address(rscratch1, 3 * BytesPerInt));
2386   __ profile_switch_case(r1, r0, r14);
2387   // continue execution
2388   __ bind(continue_execution);
2389   __ rev(r3, r3);
2390   __ add(rbcp, rbcp, r3);
2391   __ ldrb(rscratch1, Address(rbcp, 0));
2392   __ dispatch_only(vtos);
2393 }
2394 
2395 void TemplateTable::fast_binaryswitch() {
2396   transition(itos, vtos);
2397   // Implementation using the following core algorithm:
2398   //
2399   // int binary_search(int key, LookupswitchPair* array, int n) {
2400   //   // Binary search according to "Methodik des Programmierens" by
2401   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2402   //   int i = 0;
2403   //   int j = n;
2404   //   while (i+1 < j) {
2405   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2406   //     // with      Q: for all i: 0 <= i < n: key < a[i]
2407   //     // where a stands for the array and assuming that the (inexisting)
2408   //     // element a[n] is infinitely big.
2409   //     int h = (i + j) >> 1;
2410   //     // i < h < j
2411   //     if (key < array[h].fast_match()) {
2412   //       j = h;
2413   //     } else {
2414   //       i = h;
2415   //     }
2416   //   }
2417   //   // R: a[i] <= key < a[i+1] or Q
2418   //   // (i.e., if key is within array, i is the correct index)
2419   //   return i;
2420   // }
2421 
2422   // Register allocation
2423   const Register key   = r0; // already set (tosca)
2424   const Register array = r1;
2425   const Register i     = r2;
2426   const Register j     = r3;
2427   const Register h     = rscratch1;
2428   const Register temp  = rscratch2;
2429 
2430   // Find array start
2431   __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2432                                           // get rid of this
2433                                           // instruction (change
2434                                           // offsets below)
2435   __ bic(array, array, BytesPerInt - 1);
2436 
2437   // Initialize i & j
2438   __ mov(i, 0);                            // i = 0;
2439   __ ldr(j, Address(array, -BytesPerInt)); // j = length(array);
2440 
2441   // Convert j into native byteordering
2442   __ rev(j, j);
2443 
2444   // And start
2445   Label entry;
2446   __ b(entry);
2447 
2448   // binary search loop
2449   {
2450     Label loop;
2451     __ bind(loop);
2452     // int h = (i + j) >> 1;
2453     __ add(h, i, j);                           // h = i + j;
2454     __ lsr(h, h, 1);                                   // h = (i + j) >> 1;
2455     // if (key < array[h].fast_match()) {
2456     //   j = h;
2457     // } else {
2458     //   i = h;
2459     // }
2460     // Convert array[h].match to native byte-ordering before compare
2461     __ ldr(temp, Address(array, h, lsl(3)));
2462     __ rev(temp, temp);
2463     __ cmp(key, temp);
2464     // j = h if (key <  array[h].fast_match())
2465     __ mov(j, h, Assembler::LT);
2466     // i = h if (key >= array[h].fast_match())
2467     __ mov(i, h, Assembler::GE);
2468     // while (i+1 < j)
2469     __ bind(entry);
2470     __ add(h, i, 1);          // i+1
2471     __ cmp(h, j);             // i+1 < j
2472     __ b(loop, Assembler::LT);
2473   }
2474 
2475   // end of binary search, result index is i (must check again!)
2476   Label default_case;
2477   // Convert array[i].match to native byte-ordering before compare
2478   __ ldr(temp, Address(array, i, lsl(3)));
2479   __ rev(temp, temp);
2480   __ cmp(key, temp);
2481   __ b(default_case, Assembler::NE);
2482 
2483   // entry found -> j = offset
2484   __ add(j, array, i, lsl(3));
2485   __ ldr(j, Address(j, BytesPerInt));
2486   __ profile_switch_case(i, key, array);
2487   __ rev(j, j);
2488   __ load_unsigned_byte(rscratch1, Address(rbcp, j));
2489   __ lea(rbcp, Address(rbcp, j));
2490   __ dispatch_only(vtos);
2491 
2492   // default case -> j = default offset
2493   __ bind(default_case);
2494   __ profile_switch_default(i);
2495   __ ldr(j, Address(array, -2 * BytesPerInt));
2496   __ rev(j, j);
2497   __ load_unsigned_byte(rscratch1, Address(rbcp, j));
2498   __ lea(rbcp, Address(rbcp, j));
2499   __ dispatch_only(vtos);
2500 }
2501 
2502 void TemplateTable::_return(TosState state)
2503 {
2504   __ reg_printf("STARTING RETURN\n");
2505   //__ stop("_return");
2506   transition(state, state);
2507   if(ltos == state) {
2508     __ reg_printf("Doing long return, tos value is 0x%08x%08x\n", r1, r0);
2509   } else if ( itos == state || atos == state) {
2510     __ reg_printf("Doing int/ref return, tos value is 0x%08x\n", r0);
2511   }
2512 
2513   assert(_desc->calls_vm(),
2514          "inconsistent calls_vm information"); // call in remove_activation
2515 
2516   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2517     assert(state == vtos, "only valid state");
2518 
2519     __ reg_printf("A\n");
2520     __ ldr(c_rarg1, aaddress(0));
2521     __ reg_printf("object is = %p\nB\n", c_rarg1);
2522     __ load_klass(r3, c_rarg1);
2523     __ reg_printf("C\n");
2524     __ ldr(r3, Address(r3, Klass::access_flags_offset()));
2525     __ reg_printf("D\n");
2526     __ tst(r3, JVM_ACC_HAS_FINALIZER);
2527     __ reg_printf("E\n");
2528     Label skip_register_finalizer;
2529     __ b(skip_register_finalizer, Assembler::EQ);
2530     __ reg_printf("About to call into the VM\n");
2531     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2532     __ reg_printf("F\n");
2533     __ bind(skip_register_finalizer);
2534   }
2535 
2536   // Issue a StoreStore barrier after all stores but before return
2537   // from any constructor for any class with a final field.  We don't
2538   // know if this is a finalizer, so we always do so.
2539   if (_desc->bytecode() == Bytecodes::_return)
2540     __ membar(MacroAssembler::StoreStore);
2541 
2542   // Narrow result if state is itos but result type is smaller.
2543   // Need to narrow in the return bytecode rather than in generate_return_entry
2544   // since compiled code callers expect the result to already be narrowed.
2545   if (state == itos) {
2546     __ narrow(r0);
2547   }
2548 
2549   __ reg_printf("About to attmpt to remove activation with rfp = %p\n", rfp);
2550   __ remove_activation(state);
2551   __ reg_printf("Finshed _return, about to jump to lr = %p\n", lr);
2552   __ b(lr);
2553 }
2554 
2555 // ----------------------------------------------------------------------------
2556 // Volatile variables demand their effects be made known to all CPU's
2557 // in order.  Store buffers on most chips allow reads & writes to
2558 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2559 // without some kind of memory barrier (i.e., it's not sufficient that
2560 // the interpreter does not reorder volatile references, the hardware
2561 // also must not reorder them).
2562 //
2563 // According to the new Java Memory Model (JMM):
2564 // (1) All volatiles are serialized wrt to each other.  ALSO reads &
2565 //     writes act as aquire & release, so:
2566 // (2) A read cannot let unrelated NON-volatile memory refs that
2567 //     happen after the read float up to before the read.  It's OK for
2568 //     non-volatile memory refs that happen before the volatile read to
2569 //     float down below it.
2570 // (3) Similar a volatile write cannot let unrelated NON-volatile
2571 //     memory refs that happen BEFORE the write float down to after the
2572 //     write.  It's OK for non-volatile memory refs that happen after the
2573 //     volatile write to float up before it.
2574 //
2575 // We only put in barriers around volatile refs (they are expensive),
2576 // not _between_ memory refs (that would require us to track the
2577 // flavor of the previous memory refs).  Requirements (2) and (3)
2578 // require some barriers before volatile stores and after volatile
2579 // loads.  These nearly cover requirement (1) but miss the
2580 // volatile-store-volatile-load case.  This final case is placed after
2581 // volatile-stores although it could just as well go before
2582 // volatile-loads.
2583 
2584 //Note none of these calls use rscratch1, well some do but are set again before return
2585 // so index can be rscratch1 ( I think )
2586 void TemplateTable::resolve_cache_and_index(int byte_no,
2587                                             Register Rcache,
2588                                             Register index,
2589                                             size_t index_size) {
2590   // Note none of the functions called here use any rscratch
2591   // call_VM may do but will save the argument first!
2592   const Register temp = rscratch2;
2593   assert_different_registers(Rcache, index, temp);
2594 
2595   Label resolved;
2596   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2597   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2598   __ cmp(temp, (int) bytecode());  // have we resolved this bytecode?
2599   __ b(resolved, Assembler::EQ);
2600 
2601   __ reg_printf("Not resolved, resolving, with rthread = %p, rfp = %p\n", rthread, rfp);
2602   // resolve first time through
2603   address entry;
2604   switch (bytecode()) {
2605   case Bytecodes::_getstatic:
2606   case Bytecodes::_putstatic:
2607   case Bytecodes::_getfield:
2608   case Bytecodes::_putfield:
2609     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_get_put);
2610     break;
2611   case Bytecodes::_invokevirtual:
2612   case Bytecodes::_invokespecial:
2613   case Bytecodes::_invokestatic:
2614   case Bytecodes::_invokeinterface:
2615     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invoke);
2616     break;
2617   case Bytecodes::_invokehandle:
2618     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invokehandle);
2619     break;
2620   case Bytecodes::_invokedynamic:
2621     entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_invokedynamic);
2622     break;
2623   default:
2624     fatal(err_msg("unexpected bytecode: %s", Bytecodes::name(bytecode())));
2625     break;
2626   }
2627   __ mov(temp, (int) bytecode());
2628   __ call_VM(noreg, entry, temp);
2629   __ reg_printf("Resolve complete\n");
2630 
2631   // Update registers with resolved info
2632   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2633   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2634   // so all clients ofthis method must be modified accordingly
2635   __ bind(resolved);
2636 }
2637 
2638 // The Rcache and index registers must be set before call
2639 // n.b unlike x86 cache already includes the index offset
2640 void TemplateTable::load_field_cp_cache_entry(Register obj,
2641                                               Register cache,
2642                                               Register index,
2643                                               Register off,
2644                                               Register flags,
2645                                               bool is_static = false) {
2646   assert_different_registers(cache, index, flags, off);
2647 
2648   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2649   // Field offset
2650   __ ldr(off, Address(cache, in_bytes(cp_base_offset +
2651                                       ConstantPoolCacheEntry::f2_offset())));
2652   // Flags
2653   __ ldr(flags, Address(cache, in_bytes(cp_base_offset +
2654                                         ConstantPoolCacheEntry::flags_offset())));
2655 
2656   // klass overwrite register
2657   if (is_static) {
2658     __ ldr(obj, Address(cache, in_bytes(cp_base_offset +
2659                                         ConstantPoolCacheEntry::f1_offset())));
2660     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2661     __ ldr(obj, Address(obj, mirror_offset));
2662   }
2663 }
2664 
2665 
2666 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2667                                                Register method,
2668                                                Register itable_index,
2669                                                Register flags,
2670                                                bool is_invokevirtual,
2671                                                bool is_invokevfinal, /*unused*/
2672                                                bool is_invokedynamic) {
2673   // setup registers
2674   const Register cache = rscratch1;
2675   const Register index = r14;
2676   assert_different_registers(method, flags);
2677   assert_different_registers(method, cache, index);
2678   assert_different_registers(itable_index, flags);
2679   assert_different_registers(itable_index, cache, index);
2680   // determine constant pool cache field offsets
2681   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2682   const int method_offset = in_bytes(
2683     ConstantPoolCache::base_offset() +
2684       (is_invokevirtual
2685        ? ConstantPoolCacheEntry::f2_offset()
2686        : ConstantPoolCacheEntry::f1_offset()));
2687   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2688                                     ConstantPoolCacheEntry::flags_offset());
2689   // access constant pool cache fields
2690   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2691                                     ConstantPoolCacheEntry::f2_offset());
2692 
2693   size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2694   resolve_cache_and_index(byte_no, cache, index, index_size);
2695   __ ldr(method, Address(cache, method_offset));
2696 
2697   if (itable_index != noreg) {
2698     __ ldr(itable_index, Address(cache, index_offset));
2699   }
2700   __ ldr(flags, Address(cache, flags_offset));
2701 
2702   __ reg_printf("Invocation, index = %d\n", index);
2703 }
2704 
2705 
2706 // The registers cache and index expected to be set before call.
2707 // Correct values of the cache and index registers are preserved.
2708 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2709                                             bool is_static, bool has_tos) {
2710   // do the JVMTI work here to avoid disturbing the register state below
2711   // We use c_rarg registers here because we want to use the register used in
2712   // the call to the VM
2713   if (JvmtiExport::can_post_field_access()) {
2714     // Check to see if a field access watch has been set before we
2715     // take the time to call into the VM.
2716     Label L1;
2717     assert_different_registers(cache, index, r0);
2718     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2719     __ ldr(r0, Address(rscratch1));
2720     __ cmp(r0, 0);
2721     __ b(L1, Assembler::EQ);
2722 
2723     __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2724     __ lea(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2725 
2726     if (is_static) {
2727       __ mov(c_rarg1, 0); // NULL object reference
2728     } else {
2729       __ ldr(c_rarg1, at_tos()); // get object pointer without popping it
2730       __ verify_oop(c_rarg1);
2731     }
2732     // c_rarg1: object pointer or NULL
2733     // c_rarg2: cache entry pointer
2734     // c_rarg3: jvalue object on the stack
2735     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2736                                        InterpreterRuntime::post_field_access),
2737                c_rarg1, c_rarg2, c_rarg3);
2738     __ get_cache_and_index_at_bcp(cache, index, 1);
2739     __ bind(L1);
2740   }
2741 }
2742 
2743 void TemplateTable::pop_and_check_object(Register r)
2744 {
2745   __ pop_ptr(r);
2746   __ null_check(r);  // for field access must check obj.
2747   __ verify_oop(r);
2748 }
2749 
2750 void TemplateTable::getfield_or_static(int byte_no, bool is_static) {
2751   //__ stop("getfield or static");
2752   //FIXME Find a better way than this!
2753   const Register cache = r2;
2754   const Register index = r3;
2755   const Register obj   = r14;
2756   const Register off   = rscratch2; //pop_and_check_object
2757   const Register flags = r0;
2758   const Register bc    = r14; // uses same reg as obj, so don't mix them
2759 
2760   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2761   jvmti_post_field_access(cache, index, is_static, false);
2762   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2763 
2764   if (!is_static) {
2765     // obj is on the stack
2766     pop_and_check_object(obj);
2767   }
2768 
2769   const Address field(obj, off);
2770 
2771 
2772   Label Done, notByte, notBool, notInt, notShort, notChar,
2773               notLong, notFloat, notObj, notDouble;
2774 
2775   //__ bkpt(324);
2776   // x86 uses a shift and mask or wings it with a shift plus assert
2777   // the mask is not needed. aarch32 just uses bitfield extract
2778   __ extract_bits(flags, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
2779 
2780   assert(btos == 0, "change code, btos != 0");
2781   __ cbnz(flags, notByte);
2782 
2783   // btos
2784   __ load_signed_byte(r0, field);
2785   __ push(btos);
2786   // Rewrite bytecode to be faster
2787   if (!is_static) {
2788     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2789   }
2790   __ b(Done);
2791 
2792   __ bind(notByte);
2793   __ cmp(flags, ztos);
2794   __ b(notBool, Assembler::NE);
2795 
2796   // ztos (same code as btos)
2797   __ ldrsb(r0, field);
2798   __ push(ztos);
2799   // Rewrite bytecode to be faster
2800   if (!is_static) {
2801     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2802     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2803   }
2804   __ b(Done);
2805 
2806   __ bind(notBool);
2807   __ cmp(flags, atos);
2808   __ b(notObj, Assembler::NE);
2809   // atos
2810   __ load_heap_oop(r0, field);
2811   __ push(atos);
2812   __ reg_printf("Getfield or static, atos = 0x%08x\n", r0);
2813   if (!is_static) {
2814     patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2815   }
2816   __ b(Done);
2817 
2818   __ bind(notObj);
2819   __ cmp(flags, itos);
2820   __ b(notInt, Assembler::NE);
2821   // itos
2822   __ ldr(r0, field);
2823   __ push(itos);
2824   __ reg_printf("Getfield or static, itos = 0x%08x\n", r0);
2825   // Rewrite bytecode to be faster
2826   if (!is_static) {
2827     patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2828   }
2829   __ b(Done);
2830 
2831   __ bind(notInt);
2832   __ cmp(flags, ctos);
2833   __ b(notChar, Assembler::NE);
2834   // ctos
2835   __ load_unsigned_short(r0, field);
2836   __ push(ctos);
2837   // Rewrite bytecode to be faster
2838   if (!is_static) {
2839     patch_bytecode(Bytecodes::_fast_cgetfield, bc, r1);
2840   }
2841   __ b(Done);
2842 
2843   __ bind(notChar);
2844   __ cmp(flags, stos);
2845   __ b(notShort, Assembler::NE);
2846   // stos
2847   __ load_signed_short(r0, field);
2848   __ push(stos);
2849   // Rewrite bytecode to be faster
2850   if (!is_static) {
2851     patch_bytecode(Bytecodes::_fast_sgetfield, bc, r1);
2852   }
2853   __ b(Done);
2854 
2855   __ bind(notShort);
2856   __ cmp(flags, ltos);
2857   __ b(notLong, Assembler::NE);
2858   // ltos
2859   __ lea(rscratch1, field);
2860   __ atomic_ldrd(r0, r1, rscratch1);
2861   __ push(ltos);
2862   // Rewrite bytecode to be faster
2863   if (!is_static) {
2864     patch_bytecode(Bytecodes::_fast_lgetfield, bc, r1);
2865   }
2866   __ b(Done);
2867 
2868   __ bind(notLong);
2869   __ cmp(flags, ftos);
2870   __ b(notFloat, Assembler::NE);
2871   // ftos
2872   __ lea(rscratch1, field);
2873   if(hasFPU()) {
2874     __ vldr_f32(d0, Address(rscratch1));
2875   } else {
2876     __ ldr(r0, Address(rscratch1));
2877   }
2878   __ push(ftos);
2879   // Rewrite bytecode to be faster
2880   if (!is_static) {
2881     patch_bytecode(Bytecodes::_fast_fgetfield, bc, r1);
2882   }
2883   __ b(Done);
2884 
2885   __ bind(notFloat);
2886 #ifdef ASSERT
2887   __ cmp(flags, dtos);
2888   __ b(notDouble, Assembler::NE);
2889 #endif
2890   // dtos
2891   __ lea(rscratch1, field);
2892   __ atomic_ldrd(r0, r1, rscratch1);
2893   if(hasFPU()) {
2894     __ vmov_f64(d0, r0, r1);
2895   }
2896   __ push(dtos);
2897   // Rewrite bytecode to be faster
2898   if (!is_static) {
2899     patch_bytecode(Bytecodes::_fast_dgetfield, bc, r1);
2900   }
2901 #ifdef ASSERT
2902   __ b(Done);
2903 
2904   __ bind(notDouble);
2905   __ stop("Bad state");
2906 #endif
2907 
2908   __ bind(Done);
2909   // It's really not worth bothering to check whether this field
2910   // really is volatile in the slow case.
2911   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2912 }
2913 
2914 void TemplateTable::getfield(int byte_no) {
2915   getfield_or_static(byte_no, false);
2916 }
2917 
2918 void TemplateTable::getstatic(int byte_no) {
2919   getfield_or_static(byte_no, true);
2920 }
2921 
2922 // The registers cache and index expected to be set before call.
2923 // The function may destroy various registers, just not the cache and index registers.
2924 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2925   transition(vtos, vtos);
2926 
2927   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2928 
2929   if (JvmtiExport::can_post_field_modification()) {
2930     // Check to see if a field modification watch has been set before
2931     // we take the time to call into the VM.
2932     Label L1;
2933     assert_different_registers(cache, index, r0);
2934     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2935     __ ldr(r0, Address(rscratch1));
2936     __ cbz(r0, L1);
2937 
2938     __ get_cache_and_index_at_bcp(c_rarg2, rscratch1, 1);
2939 
2940     if (is_static) {
2941       // Life is simple.  Null out the object pointer.
2942       __ mov(c_rarg1, 0);
2943     } else {
2944       // Life is harder. The stack holds the value on top, followed by
2945       // the object.  We don't know the size of the value, though; it
2946       // could be one or two words depending on its type. As a result,
2947       // we must find the type to determine where the object is.
2948       __ ldr(c_rarg3, Address(c_rarg2,
2949                               in_bytes(cp_base_offset +
2950                                        ConstantPoolCacheEntry::flags_offset())));
2951       __ lsr(c_rarg3, c_rarg3,
2952              ConstantPoolCacheEntry::tos_state_shift);
2953       ConstantPoolCacheEntry::verify_tos_state_shift();
2954       Label nope2, done, ok;
2955       __ ldr(c_rarg1, at_tos_p1());  // initially assume a one word jvalue
2956       __ cmp(c_rarg3, ltos);
2957       __ b(ok, Assembler::EQ);
2958       __ cmp(c_rarg3, dtos);
2959       __ b(nope2, Assembler::NE);
2960       __ bind(ok);
2961       __ ldr(c_rarg1, at_tos_p2()); // ltos (two word jvalue)
2962       __ bind(nope2);
2963     }
2964     // cache entry pointer
2965     __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2966     // object (tos)
2967     __ mov(c_rarg3, sp);
2968     // c_rarg1: object pointer set up above (NULL if static)
2969     // c_rarg2: cache entry pointer
2970     // c_rarg3: jvalue object on the stack
2971     __ call_VM(noreg,
2972                CAST_FROM_FN_PTR(address,
2973                                 InterpreterRuntime::post_field_modification),
2974                c_rarg1, c_rarg2, c_rarg3);
2975     __ get_cache_and_index_at_bcp(cache, index, 1);
2976     __ bind(L1);
2977   }
2978 }
2979 
2980 void TemplateTable::putfield_or_static(int byte_no, bool is_static) {
2981   transition(vtos, vtos);
2982   const Register cache = r2;
2983   const Register index = rscratch1;
2984   const Register obj   = r2;
2985   const Register off   = rscratch2;
2986   const Register flags = r14;
2987   const Register bc    = r3;
2988 
2989   resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2990   __ reg_printf("Putfield or static, index = %d\n", index);
2991   jvmti_post_field_mod(cache, index, is_static);
2992   load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2993 
2994   Label Done;
2995   {
2996     Label notVolatile;
2997     __ tbz(flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2998     __ membar(MacroAssembler::StoreStore);
2999     __ bind(notVolatile);
3000   }
3001   __ reg_printf("Putfield or static B\n");
3002 
3003   // field address
3004   const Address field(obj, off);
3005 
3006   Label notByte, notBool, notInt, notShort, notChar,
3007         notLong, notFloat, notObj, notDouble;
3008 
3009   // x86 uses a shift and mask or wings it with a shift plus assert
3010   // the mask is not needed. aarch32 just uses bitfield extract
3011   __ extract_bits(rscratch1, flags, ConstantPoolCacheEntry::tos_state_shift, ConstantPoolCacheEntry::tos_state_bits);
3012 
3013   // Move the flags to rscratch2
3014 
3015   __ cmp(rscratch1, btos);
3016   __ b(notByte, Assembler::NE);
3017 
3018   // btos
3019   {
3020     __ pop(btos);
3021     if (!is_static) {
3022       pop_and_check_object(obj);
3023     }
3024     __ strb(r0, field);
3025     if (!is_static) {
3026       patch_bytecode(Bytecodes::_fast_bputfield, bc, r1, true, byte_no);
3027     }
3028     __ b(Done);
3029   }
3030 
3031   __ bind(notByte);
3032   __ cmp(rscratch1, ztos);
3033   __ b(notBool, Assembler::NE);
3034 
3035   // ztos
3036   {
3037     __ pop(ztos);
3038     if (!is_static) pop_and_check_object(obj);
3039     __ andr(r0, r0, 0x1);
3040     __ strb(r0, field);
3041     if (!is_static) {
3042       patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
3043     }
3044     __ b(Done);
3045   }
3046 
3047   __ bind(notBool);
3048   __ cmp(rscratch1, atos);
3049   __ b(notObj, Assembler::NE);
3050 
3051   // atos
3052   {
3053     __ pop(atos);
3054     if (!is_static) {
3055       pop_and_check_object(obj);
3056     }
3057     // Store into the field
3058     do_oop_store(_masm, field, r0, _bs->kind(), false);
3059     if (!is_static) {
3060       patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
3061     }
3062     __ b(Done);
3063   }
3064 
3065   __ bind(notObj);
3066   __ cmp(rscratch1, itos);
3067   __ b(notInt, Assembler::NE);
3068 
3069   // itos
3070   {
3071     __ pop(itos);
3072     if (!is_static) {
3073       pop_and_check_object(obj);
3074     }
3075     __ str(r0, field);
3076     if (!is_static) {
3077       patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
3078     }
3079     __ b(Done);
3080   }
3081 
3082   __ bind(notInt);
3083   __ cmp(rscratch1, ctos);
3084   __ b(notChar, Assembler::NE);
3085 
3086   // ctos
3087   {
3088     __ pop(ctos);
3089     if (!is_static) {
3090       pop_and_check_object(obj);
3091     }
3092     __ strh(r0, field);
3093     if (!is_static) {
3094       patch_bytecode(Bytecodes::_fast_cputfield, bc, r1, true, byte_no);
3095     }
3096     __ b(Done);
3097   }
3098 
3099   __ bind(notChar);
3100   __ cmp(rscratch1, stos);
3101   __ b(notShort, Assembler::NE);
3102 
3103   // stos
3104   {
3105     __ pop(stos);
3106     if (!is_static) {
3107       pop_and_check_object(obj);
3108     }
3109     __ strh(r0, field);
3110     if (!is_static) {
3111       patch_bytecode(Bytecodes::_fast_sputfield, bc, r1, true, byte_no);
3112     }
3113     __ b(Done);
3114   }
3115 
3116   __ bind(notShort);
3117   __ cmp(rscratch1, ltos);
3118   __ b(notLong, Assembler::NE);
3119 
3120   // ltos
3121   {
3122     __ pop(ltos);
3123     if (!is_static) {
3124       pop_and_check_object(obj);
3125     }
3126     __ lea(rscratch1, field);
3127     __ atomic_strd(r0, r1, rscratch1, r2, r3);
3128     if (!is_static) {
3129       patch_bytecode(Bytecodes::_fast_lputfield, bc, r1, true, byte_no);
3130     }
3131     __ b(Done);
3132   }
3133 
3134   __ bind(notLong);
3135   __ cmp(rscratch1, ftos);
3136   __ b(notFloat, Assembler::NE);
3137 
3138   // ftos
3139   {
3140     __ pop(ftos);
3141     if (!is_static) {
3142       pop_and_check_object(obj);
3143     }
3144     __ lea(rscratch1, field);
3145     if(hasFPU()) {
3146         __ vstr_f32(d0, Address(rscratch1));
3147     } else {
3148         __ str(r0, Address(rscratch1));
3149     }
3150     if (!is_static) {
3151       patch_bytecode(Bytecodes::_fast_fputfield, bc, r1, true, byte_no);
3152     }
3153     __ b(Done);
3154   }
3155 
3156   __ bind(notFloat);
3157 #ifdef ASSERT
3158   __ cmp(rscratch1, dtos);
3159   __ b(notDouble, Assembler::NE);
3160 #endif // ASSERT
3161 
3162   // dtos
3163   {
3164     __ pop(dtos);
3165     if (!is_static) {
3166       pop_and_check_object(obj);
3167     }
3168     __ lea(rscratch1, field);
3169     if(hasFPU()) {
3170         __ vmov_f64(r0, r1, d0);
3171     }
3172     __ atomic_strd(r0, r1, rscratch1, r2, r3);
3173     if (!is_static) {
3174       patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
3175     }
3176   }
3177 
3178 #ifdef ASSERT
3179   __ b(Done);
3180 
3181   __ bind(notDouble);
3182   __ stop("Bad state");
3183 #endif // ASSERT
3184 
3185   __ bind(Done);
3186 
3187   {
3188     Label notVolatile;
3189     __ tbz(flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3190     __ membar(MacroAssembler::StoreLoad);
3191     __ bind(notVolatile);
3192   }
3193   //FIXME find a more elegant way!
3194   __ get_dispatch();
3195 }
3196 
3197 void TemplateTable::putfield(int byte_no) {
3198   putfield_or_static(byte_no, false);
3199 }
3200 
3201 void TemplateTable::putstatic(int byte_no) {
3202   putfield_or_static(byte_no, true);
3203 }
3204 
3205 void TemplateTable::jvmti_post_fast_field_mod()
3206 {
3207   if (JvmtiExport::can_post_field_modification()) {
3208     // Check to see if a field modification watch has been set before
3209     // we take the time to call into the VM.
3210     Label L2;
3211     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
3212     __ ldr(c_rarg3, Address(rscratch1));
3213     __ cmp(c_rarg3, 0);
3214     __ b(L2, Assembler::EQ);
3215     __ pop_ptr(r14);                  // copy the object pointer from tos
3216     __ verify_oop(r14);
3217     __ push_ptr(r14);                 // put the object pointer back on tos
3218     // Save tos values before call_VM() clobbers them. Since we have
3219     // to do it for every data type, we use the saved values as the
3220     // jvalue object.
3221     switch (bytecode()) {          // load values into the jvalue object
3222     case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
3223     case Bytecodes::_fast_bputfield: // fall through
3224     case Bytecodes::_fast_zputfield: // fall through
3225     case Bytecodes::_fast_sputfield: // fall through
3226     case Bytecodes::_fast_cputfield: // fall through
3227     case Bytecodes::_fast_iputfield: __ push_i(r0); break;
3228     case Bytecodes::_fast_dputfield:
3229         if(hasFPU()) {
3230             __ push_d();
3231         } else {
3232             __ push_l();
3233         }
3234         break;
3235     case Bytecodes::_fast_fputfield:
3236         if(hasFPU()) {
3237             __ push_f();
3238         } else {
3239             __ push_i();
3240         }
3241         break;
3242     case Bytecodes::_fast_lputfield: __ push_l(r0); break;
3243 
3244     default:
3245       ShouldNotReachHere();
3246     }
3247     __ mov(c_rarg3, sp);             // points to jvalue on the stack
3248     // access constant pool cache entry
3249     __ get_cache_entry_pointer_at_bcp(c_rarg2, r0, 1);
3250     __ verify_oop(r14);
3251     // r14: object pointer copied above
3252     // c_rarg2: cache entry pointer
3253     // c_rarg3: jvalue object on the stack
3254     __ call_VM(noreg,
3255                CAST_FROM_FN_PTR(address,
3256                                 InterpreterRuntime::post_field_modification),
3257                r14, c_rarg2, c_rarg3);
3258 
3259     switch (bytecode()) {             // restore tos values
3260     case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
3261     case Bytecodes::_fast_fputfield:
3262         if(hasFPU()) {
3263             __ pop_f(); break;
3264         }
3265     case Bytecodes::_fast_bputfield: // fall through
3266     case Bytecodes::_fast_zputfield: // fall through
3267     case Bytecodes::_fast_sputfield: // fall through
3268     case Bytecodes::_fast_cputfield: // fall through
3269     case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
3270     case Bytecodes::_fast_dputfield:
3271         if(hasFPU()) {
3272             __ pop_d(); break;
3273         }
3274     case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
3275     }
3276     __ bind(L2);
3277   }
3278 }
3279 
3280 void TemplateTable::fast_storefield(TosState state)
3281 {
3282   transition(state, vtos);
3283 
3284   ByteSize base = ConstantPoolCache::base_offset();
3285 
3286   jvmti_post_fast_field_mod();
3287 
3288   // access constant pool cache
3289   __ get_cache_and_index_at_bcp(r2, rscratch1, 1); // index not used
3290 
3291   // test for volatile with r14
3292   __ ldr(r14, Address(r2, in_bytes(base +
3293                                    ConstantPoolCacheEntry::flags_offset())));
3294 
3295   // replace index with field offset from cache entry
3296   __ ldr(r3, Address(r2, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
3297 
3298   {
3299     Label notVolatile;
3300     __ tbz(r14, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3301     __ membar(MacroAssembler::StoreStore);
3302     __ bind(notVolatile);
3303   }
3304 
3305   Label notVolatile;
3306 
3307   // Get object from stack
3308   pop_and_check_object(r2);
3309 
3310   // field address
3311   const Address field(r2, r3);
3312 
3313   // access field
3314   switch (bytecode()) {
3315   case Bytecodes::_fast_aputfield:
3316     do_oop_store(_masm, field, r0, _bs->kind(), false);
3317     break;
3318   case Bytecodes::_fast_dputfield:
3319     if(hasFPU()) {
3320         __ vmov_f64(r0, r1, d0);
3321     }
3322   case Bytecodes::_fast_lputfield:
3323     __ lea(rscratch1, field);
3324     __ atomic_strd(r0, r1, rscratch1, r2, r3);
3325     break;
3326   case Bytecodes::_fast_fputfield:
3327     if(hasFPU()) {
3328     __ lea(rscratch1, field);
3329     __ vstr_f32(d0, Address(rscratch1));
3330     break;
3331     }
3332   case Bytecodes::_fast_iputfield:
3333     __ str(r0, field);
3334     break;
3335   case Bytecodes::_fast_zputfield:
3336     __ andr(r0, r0, 0x1);  // boolean is true if LSB is 1
3337     // fall through to bputfield
3338   case Bytecodes::_fast_bputfield:
3339     __ strb(r0, field);
3340     break;
3341   case Bytecodes::_fast_sputfield:
3342     // fall through
3343   case Bytecodes::_fast_cputfield:
3344     __ strh(r0, field);
3345     break;
3346   default:
3347     ShouldNotReachHere();
3348   }
3349 
3350   {
3351     Label notVolatile;
3352     __ tbz(r14, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3353     __ membar(MacroAssembler::StoreLoad);
3354     __ bind(notVolatile);
3355   }
3356 }
3357 
3358 
3359 void TemplateTable::fast_accessfield(TosState state)
3360 {
3361   transition(atos, state);
3362   // Do the JVMTI work here to avoid disturbing the register state below
3363   if (JvmtiExport::can_post_field_access()) {
3364     // Check to see if a field access watch has been set before we
3365     // take the time to call into the VM.
3366     Label L1;
3367     __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
3368     __ ldr(r2, Address(rscratch1));
3369     __ cmp(r2, 0);
3370     __ b(L1, Assembler::EQ);
3371     // access constant pool cache entry
3372     __ get_cache_entry_pointer_at_bcp(c_rarg2, rscratch2, 1);
3373     __ verify_oop(r0);
3374     __ push_ptr(r0);  // save object pointer before call_VM() clobbers it
3375     __ mov(c_rarg1, r0);
3376     // c_rarg1: object pointer copied above
3377     // c_rarg2: cache entry pointer
3378     __ call_VM(noreg,
3379                CAST_FROM_FN_PTR(address,
3380                                 InterpreterRuntime::post_field_access),
3381                c_rarg1, c_rarg2);
3382     __ pop_ptr(r0); // restore object pointer
3383     __ bind(L1);
3384   }
3385 
3386   // access constant pool cache
3387   __ get_cache_and_index_at_bcp(r2, r1, 1);
3388   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3389                                   ConstantPoolCacheEntry::f2_offset())));
3390   __ ldr(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3391                                   ConstantPoolCacheEntry::flags_offset())));
3392 
3393   // r0: object
3394   __ verify_oop(r0);
3395   __ null_check(r0);
3396   const Address field(r0, r1);
3397 
3398   // access field
3399   switch (bytecode()) {
3400   case Bytecodes::_fast_agetfield:
3401     __ load_heap_oop(r0, field);
3402     __ verify_oop(r0);
3403     break;
3404   case Bytecodes::_fast_dgetfield:
3405     if(hasFPU()) {
3406     __ lea(rscratch1, field); // r0 <= field
3407     __ atomic_ldrd(r0, r1, rscratch1);
3408     __ vmov_f64(d0, r0, r1);
3409     break;
3410     }
3411   case Bytecodes::_fast_lgetfield:
3412     __ lea(rscratch1, field);
3413     __ atomic_ldrd(r0, r1, rscratch1);
3414     break;
3415   case Bytecodes::_fast_fgetfield:
3416     if(hasFPU()) {
3417     __ lea(r0, field); // r0 <= field
3418     __ vldr_f32(d0, Address(r0));
3419       break;
3420     }
3421   case Bytecodes::_fast_igetfield:
3422     __ ldr(r0, field);
3423     break;
3424   case Bytecodes::_fast_bgetfield:
3425     __ load_signed_byte(r0, field);
3426     break;
3427   case Bytecodes::_fast_sgetfield:
3428     __ load_signed_short(r0, field);
3429     break;
3430   case Bytecodes::_fast_cgetfield:
3431     __ load_unsigned_short(r0, field);
3432     break;
3433   default:
3434     ShouldNotReachHere();
3435   }
3436   {
3437     Label notVolatile;
3438     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3439     __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3440     __ bind(notVolatile);
3441   }
3442 }
3443 
3444 void TemplateTable::fast_xaccess(TosState state)
3445 {
3446   transition(vtos, state);
3447 
3448   // get receiver
3449   __ ldr(r0, aaddress(0));
3450   // access constant pool cache
3451   __ get_cache_and_index_at_bcp(r2, r3, 2);
3452   __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3453                                   ConstantPoolCacheEntry::f2_offset())));
3454   // make sure exception is reported in correct bcp range (getfield is
3455   // next instruction)
3456   __ add(rbcp, rbcp, 1);
3457   __ null_check(r0);
3458 
3459   Address field(r0, r1);
3460   switch (state) {
3461   case ftos:
3462     if(hasFPU()) {
3463     __ lea(r0, field);
3464     __ vldr_f32(d0, Address(r0));
3465     break;
3466     }
3467   case itos:
3468     __ ldr(r0, field);
3469     break;
3470   case atos:
3471     __ load_heap_oop(r0, field);
3472     __ verify_oop(r0);
3473     break;
3474   default:
3475     ShouldNotReachHere();
3476   }
3477 
3478   {
3479     Label notVolatile;
3480     __ ldr(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3481                                      ConstantPoolCacheEntry::flags_offset())));
3482     __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3483     __ membar(MacroAssembler::LoadLoad);
3484     __ bind(notVolatile);
3485   }
3486 
3487   __ sub(rbcp, rbcp, 1);
3488 }
3489 
3490 
3491 
3492 //-----------------------------------------------------------------------------
3493 // Calls
3494 
3495 void TemplateTable::count_calls(Register method, Register temp) {
3496   // implemented elsewhere
3497   ShouldNotReachHere();
3498 }
3499 
3500 void TemplateTable::prepare_invoke(int byte_no,
3501                                    Register method, // linked method (or i-klass)
3502                                    Register index,  // itable index, MethodType, etc.
3503                                    Register recv,   // if caller wants to see it
3504                                    Register flags   // if caller wants to test it
3505                                    ) {
3506   // determine flags
3507   Bytecodes::Code code = bytecode();
3508   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
3509   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
3510   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
3511   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
3512   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
3513   const bool load_receiver       = (recv  != noreg);
3514   const bool save_flags          = (flags != noreg);
3515   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3516   assert(save_flags    == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3517   assert(flags == noreg || flags == r3, "");
3518   assert(recv  == noreg || recv  == r2, "");
3519 
3520   // setup registers & access constant pool cache
3521   if (recv  == noreg)  recv  = r2;
3522   if (flags == noreg)  flags = r3;
3523   assert_different_registers(method, index, recv, flags);
3524 
3525   // save 'interpreter return address'
3526   __ save_bcp();
3527 
3528   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3529 
3530   // maybe push appendix to arguments (just before return address)
3531   if (is_invokedynamic || is_invokehandle) {
3532     Label L_no_push;
3533     __ tbz(flags, ConstantPoolCacheEntry::has_appendix_shift, L_no_push);
3534     // Push the appendix as a trailing parameter.
3535     // This must be done before we get the receiver,
3536     // since the parameter_size includes it.
3537     __ push(r14); //NOT NEEDED?!
3538     __ mov(r14, index);
3539     assert(ConstantPoolCacheEntry::_indy_resolved_references_appendix_offset == 0, "appendix expected at index+0");
3540     __ load_resolved_reference_at_index(index, r14);
3541     __ pop(r14);
3542     __ push(index);  // push appendix (MethodType, CallSite, etc.)
3543     __ bind(L_no_push);
3544   }
3545 
3546   // load receiver if needed (note: no return address pushed yet)
3547   if (load_receiver) {
3548     __ andr(recv, flags, ConstantPoolCacheEntry::parameter_size_mask);
3549     // const int no_return_pc_pushed_yet = -1;  // argument slot correction before we push return address
3550     // const int receiver_is_at_end      = -1;  // back off one slot to get receiver
3551     // Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
3552     // __ movptr(recv, recv_addr);
3553 
3554     __ add(rscratch1, sp, recv, lsl(2));
3555     __ ldr(recv, Address(rscratch1, -Interpreter::expr_offset_in_bytes(1)));
3556     __ verify_oop(recv);
3557   }
3558 
3559   // compute return type
3560   // x86 uses a shift and mask or wings it with a shift plus assert
3561   // the mask is not needed. aarch32 just uses bitfield extract
3562   __ extract_bits(rscratch2, flags, ConstantPoolCacheEntry::tos_state_shift,  ConstantPoolCacheEntry::tos_state_bits);
3563   // load return address
3564   {
3565     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3566     __ mov(rscratch1, table_addr);
3567     __ ldr(lr, Address(rscratch1, rscratch2, lsl(2)));
3568   }
3569 }
3570 
3571 
3572 void TemplateTable::invokevirtual_helper(Register index,
3573                                          Register recv,
3574                                          Register flags)
3575 {
3576   // Uses temporary registers r0, r3
3577   assert_different_registers(index, recv, r0, r3);
3578   // Test for an invoke of a final method
3579   Label notFinal;
3580   __ tbz(flags, ConstantPoolCacheEntry::is_vfinal_shift, notFinal);
3581 
3582   __ reg_printf("It's a virtual final call\n");
3583   const Register method = index;  // method must be rmethod
3584   assert(method == rmethod,
3585          "methodOop must be rmethod for interpreter calling convention");
3586 
3587   // do the call - the index is actually the method to call
3588   // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3589 
3590   // It's final, need a null check here!
3591   __ null_check(recv);
3592 
3593   // profile this call
3594   __ profile_final_call(r0);
3595   __ profile_arguments_type(r0, method, r14, true);
3596 
3597   __ jump_from_interpreted(method, r0);
3598 
3599   __ bind(notFinal);
3600   __ reg_printf("It's not a virtual final call\n");
3601   // get receiver klass
3602   __ null_check(recv, oopDesc::klass_offset_in_bytes());
3603   __ load_klass(r0, recv);
3604 
3605   // profile this call
3606   __ profile_virtual_call(r0, rlocals, r3);
3607 
3608   // get target methodOop & entry point
3609   __ lookup_virtual_method(r0, index, method);
3610   __ profile_arguments_type(r3, method, r14, true);
3611 
3612   __ jump_from_interpreted(method, r3);
3613 }
3614 
3615 void TemplateTable::invokevirtual(int byte_no)
3616 {
3617   transition(vtos, vtos);
3618   assert(byte_no == f2_byte, "use this argument");
3619 
3620   __ reg_printf("Invokevirtual, the sp is %p\n", sp);
3621   prepare_invoke(byte_no, rmethod, noreg, r2, r3);
3622 
3623   // rmethod: index (actually a Method*)
3624   // r2: receiver
3625   // r3: flags
3626 
3627   invokevirtual_helper(rmethod, r2, r3);
3628 }
3629 
3630 void TemplateTable::invokespecial(int byte_no)
3631 {
3632   transition(vtos, vtos);
3633   assert(byte_no == f1_byte, "use this argument");
3634   __ ldr(rscratch1, Address(sp));
3635   __ reg_printf("Stack pointer is %p, tos word = %p\n", sp, rscratch1);
3636 
3637   prepare_invoke(byte_no, rmethod, noreg,  // get f1 Method*
3638                  r2);  // get receiver also for null check
3639 
3640   __ verify_oop(r2);
3641   __ null_check(r2);
3642 
3643   // do the call
3644   __ profile_call(r0);
3645   __ profile_arguments_type(r0, rmethod, rbcp, false);
3646   __ jump_from_interpreted(rmethod, r0);
3647 }
3648 
3649 void TemplateTable::invokestatic(int byte_no)
3650 {
3651   transition(vtos, vtos);
3652   assert(byte_no == f1_byte, "use this argument");
3653 
3654   prepare_invoke(byte_no, rmethod);  // get f1 Method*
3655   // do the call
3656   __ profile_call(r0);
3657   __ profile_arguments_type(r0, rmethod, r14, false);
3658   __ jump_from_interpreted(rmethod, r0);
3659 }
3660 
3661 void TemplateTable::fast_invokevfinal(int byte_no) {
3662   transition(vtos, vtos);
3663   assert(byte_no == f2_byte, "use this argument");
3664   __ stop("fast_invokevfinal not used on aarch32");}
3665 
3666 void TemplateTable::invokeinterface(int byte_no) {
3667   transition(vtos, vtos);
3668   assert(byte_no == f1_byte, "use this argument");
3669 
3670   Register temp = rdispatch; //free at this point and reloaded later
3671   prepare_invoke(byte_no, r0, rmethod,  // get f1 Klass*, f2 Method*
3672                  r2, r3); // recv, flags
3673 
3674   // r0: interface klass (from f1)
3675   // rmethod: method (from f2)
3676   // r2: receiver
3677   // r3: flags
3678 
3679   // Special case of invokeinterface called for virtual method of
3680   // java.lang.Object.  See cpCacheOop.cpp for details.
3681   // This code isn't produced by javac, but could be produced by
3682   // another compliant java compiler.
3683   Label notMethod;
3684   __ tbz(r3, ConstantPoolCacheEntry::is_forced_virtual_shift, notMethod);
3685 
3686   __ reg_printf("ABC: Invoking invokevirtual_helper\n");
3687   invokevirtual_helper(rmethod, r2, r3); //loads lr too
3688   __ bind(notMethod);
3689 
3690   __ reg_printf("ABC: invokeinterface says 'It's not a method'\n");
3691   // Get receiver klass into r3 - also a null check
3692   __ restore_locals();
3693   __ null_check(r2, oopDesc::klass_offset_in_bytes());
3694   __ load_klass(r3, r2);
3695 
3696   Label no_such_interface, no_such_method;
3697 
3698   // Receiver subtype check against REFC.
3699   // Superklass in r0. Subklass in r3.
3700   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3701                              r3, r0, noreg,
3702                              // outputs: scan temp. reg, scan temp. reg
3703                              rbcp, temp,
3704                              no_such_interface,
3705                              /*return_method=*/false);
3706 
3707 
3708   // profile this call
3709   __ restore_bcp(); // rbcp was destroyed by receiver type check
3710   __ profile_virtual_call(r3, temp, r1);
3711 
3712   // Get declaring interface class from method, and itable index
3713   __ ldr(r0, Address(rmethod, Method::const_offset()));
3714   __ ldr(r0, Address(r0, ConstMethod::constants_offset()));
3715   __ ldr(r0, Address(r0, ConstantPool::pool_holder_offset_in_bytes()));
3716   __ ldr(rmethod, Address(rmethod, Method::itable_index_offset()));
3717   assert(Method::itable_index_max <= 0, "incorrect below");
3718   __ add(temp, rmethod, -Method::itable_index_max);
3719   __ neg(rmethod, temp);
3720 
3721   __ lookup_interface_method(// inputs: rec. class, interface, itable index
3722                              r3, r0, rmethod,
3723                              // outputs: method, scan temp. reg
3724                              rmethod, temp,
3725                              no_such_interface);
3726 
3727   // rmethod,: methodOop to call
3728   // r2: receiver
3729   // Check for abstract method error
3730   // Note: This should be done more efficiently via a throw_abstract_method_error
3731   //       interpreter entry point and a conditional jump to it in case of a null
3732   //       method.
3733   __ cbz(rmethod, no_such_method);
3734 
3735   __ profile_arguments_type(r3, rmethod, temp, true);
3736 
3737   // do the call
3738   // r2: receiver
3739   // rmethod,: methodOop
3740   __ jump_from_interpreted(rmethod, r3);
3741   __ should_not_reach_here();
3742 
3743   // exception handling code follows...
3744   // note: must restore interpreter registers to canonical
3745   //       state for exception handling to work correctly!
3746 
3747   __ bind(no_such_method);
3748   __ reg_printf("ABC: invokeinterface says 'There's no such method'\n");
3749   // throw exception
3750   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3751   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3752   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
3753   // the call_VM checks for exception, so we should never return here.
3754   __ should_not_reach_here();
3755 
3756   __ bind(no_such_interface);
3757   __ reg_printf("ABC: invokeinterface says 'There's no such interface'\n");
3758   // throw exception
3759   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
3760   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
3761   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3762                    InterpreterRuntime::throw_IncompatibleClassChangeError));
3763   // the call_VM checks for exception, so we should never return here.
3764   __ should_not_reach_here();
3765   return;
3766 }
3767 
3768 void TemplateTable::invokehandle(int byte_no) {
3769   transition(vtos, vtos);
3770   assert(byte_no == f1_byte, "use this argument");
3771 
3772   prepare_invoke(byte_no, rmethod, r0, r2);
3773   __ verify_method_ptr(r2);
3774   __ verify_oop(r2);
3775   __ null_check(r2);
3776 
3777   // FIXME: profile the LambdaForm also
3778 
3779   __ profile_final_call(r14);
3780   __ profile_arguments_type(r14, rmethod, rscratch2, true);
3781 
3782   __ jump_from_interpreted(rmethod, r0);
3783 }
3784 
3785 void TemplateTable::invokedynamic(int byte_no) {
3786   transition(vtos, vtos);
3787   assert(byte_no == f1_byte, "use this argument");
3788 
3789   prepare_invoke(byte_no, rmethod, r0);
3790 
3791   // r0: CallSite object (from cpool->resolved_references[])
3792   // rmethod: MH.linkToCallSite method (from f2)
3793 
3794   // Note:  r0_callsite is already pushed by prepare_invoke
3795 
3796   // %%% should make a type profile for any invokedynamic that takes a ref argument
3797   // profile this call
3798   __ profile_call(rbcp);
3799   __ profile_arguments_type(r3, rmethod, rscratch2, false);
3800 
3801   __ verify_oop(r0);
3802 
3803   __ jump_from_interpreted(rmethod, r0);
3804 }
3805 
3806 
3807 //-----------------------------------------------------------------------------
3808 // Allocation
3809 
3810 void TemplateTable::_new() {
3811   transition(vtos, atos);
3812 
3813   __ get_unsigned_2_byte_index_at_bcp(r3, 1);
3814   Label slow_case;
3815   Label done;
3816   Label initialize_header;
3817   Label initialize_object; // including clearing the fields
3818   Label allocate_shared;
3819 
3820   __ get_cpool_and_tags(r2, r0);
3821   // Make sure the class we're about to instantiate has been resolved.
3822   // This is done before loading InstanceKlass to be consistent with the order
3823   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3824   const int tags_offset = Array<u1>::base_offset_in_bytes();
3825   __ lea(rscratch1, Address(r0, r3, lsl(0)));
3826   __ ldrb(rscratch1, Address(rscratch1, tags_offset));
3827   __ cmp(rscratch1, JVM_CONSTANT_Class);
3828   __ b(slow_case, Assembler::NE);
3829 
3830   // get InstanceKlass
3831   __ lea(r2, Address(r2, r3, lsl(2)));
3832   __ ldr(r2, Address(r2, sizeof(ConstantPool)));
3833 
3834   // make sure klass is initialized & doesn't have finalizer
3835   // make sure klass is fully initialized
3836   __ ldrb(rscratch1, Address(r2, InstanceKlass::init_state_offset()));
3837   __ cmp(rscratch1, InstanceKlass::fully_initialized);
3838   __ b(slow_case, Assembler::NE);
3839 
3840   // get instance_size in InstanceKlass (scaled to a count of bytes)
3841   __ ldr(r3, Address(r2, Klass::layout_helper_offset()));
3842   // test to see if it has a finalizer or is malformed in some way
3843   __ tbnz(r3, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
3844 
3845   // Allocate the instance
3846   // 1) Try to allocate in the TLAB
3847   // 2) if fail and the object is large allocate in the shared Eden
3848   // 3) if the above fails (or is not applicable), go to a slow case
3849   // (creates a new TLAB, etc.)
3850 
3851   const bool allow_shared_alloc =
3852     !CMSIncrementalMode && Universe::heap()->supports_inline_contig_alloc();
3853 
3854   if (UseTLAB) {
3855     __ tlab_allocate(r0, r3, 0, noreg, r1,
3856                      allow_shared_alloc ? allocate_shared : slow_case);
3857 
3858     if (ZeroTLAB) {
3859       // the fields have been already cleared
3860       __ b(initialize_header);
3861     } else {
3862       // initialize both the header and fields
3863       __ b(initialize_object);
3864     }
3865   }
3866 
3867   // Allocation in the shared Eden, if allowed.
3868   //
3869   // r3: instance size in bytes
3870   if (allow_shared_alloc) {
3871     __ bind(allocate_shared);
3872     __ eden_allocate(r0, r3, 0, r14, slow_case);
3873     __ incr_allocated_bytes(rthread, r3, 0, rscratch1);
3874   }
3875 
3876   if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) {
3877     // The object is initialized before the header.  If the object size is
3878     // zero, go directly to the header initialization.
3879     __ bind(initialize_object);
3880     __ sub(r3, r3, sizeof(oopDesc));
3881     __ cbz(r3, initialize_header);
3882 
3883     // Initialize object fields
3884     {
3885       __ add(rscratch1, r0, sizeof(oopDesc));
3886       __ mov(rscratch2, 0);
3887       Label loop;
3888       __ bind(loop);
3889       __ str(rscratch2, Address(__ post(rscratch1, BytesPerInt)));
3890       __ sub(r3, r3, BytesPerInt);
3891       __ cbnz(r3, loop);
3892     }
3893 
3894     // initialize object header only.
3895     __ bind(initialize_header);
3896     if (UseBiasedLocking) {
3897       __ ldr(rscratch1, Address(r2, Klass::prototype_header_offset()));
3898     } else {
3899       __ mov(rscratch1, (intptr_t)markOopDesc::prototype());
3900     }
3901     __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
3902     __ mov(rscratch2, 0);
3903     __ store_klass_gap(r0, rscratch2);  // zero klass gap for compressed oops - not using
3904     // not using compressed oops
3905     __ store_klass(r0, r2);      // store klass last
3906 
3907 #ifdef DTRACE_ENABLED
3908     {
3909       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3910       // Trigger dtrace event for fastpath
3911       __ push(atos); // save the return value
3912       __ call_VM_leaf(
3913            CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), r0);
3914       __ pop(atos); // restore the return value
3915 
3916     }
3917 #endif
3918     __ b(done);
3919   }
3920 
3921   // slow case
3922   __ bind(slow_case);
3923   __ get_constant_pool(c_rarg1);
3924   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3925   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3926   __ verify_oop(r0);
3927 
3928   // continue
3929   __ bind(done);
3930 
3931   __ reg_printf("New object reference is %p\n", r0);
3932   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3933   __ membar(Assembler::StoreStore);
3934 }
3935 
3936 void TemplateTable::newarray() {
3937   transition(itos, atos);
3938   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3939   __ mov(c_rarg2, r0);
3940   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3941           c_rarg1, c_rarg2);
3942   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3943   __ membar(Assembler::StoreStore);
3944 }
3945 
3946 void TemplateTable::anewarray() {
3947   transition(itos, atos);
3948   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3949   __ reg_printf("Index = %d\n", c_rarg2);
3950   __ get_constant_pool(c_rarg1);
3951   __ mov(c_rarg3, r0);
3952   __ reg_printf("About to call InterpreterRuntime::anewarray\n");
3953   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3954           c_rarg1, c_rarg2, c_rarg3);
3955   __ reg_printf("Finshed call to InterpreterRuntime::anewarray\n");
3956   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3957   __ membar(Assembler::StoreStore);
3958   __ reg_printf("Finshed anewarray\n");
3959 }
3960 
3961 void TemplateTable::arraylength() {
3962   transition(atos, itos);
3963   __ null_check(r0, arrayOopDesc::length_offset_in_bytes());
3964   __ ldr(r0, Address(r0, arrayOopDesc::length_offset_in_bytes()));
3965 }
3966 
3967 void TemplateTable::checkcast()
3968 {
3969   transition(atos, atos);
3970   Label done, is_null, ok_is_subtype, quicked, resolved;
3971   __ cbz(r0, is_null);
3972 
3973   // Get cpool & tags index
3974   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3975   __ get_unsigned_2_byte_index_at_bcp(r14, 1); // r14=index
3976   // See if bytecode has already been quicked
3977   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3978   __ ldrb(r1, Address(rscratch1, r14));
3979   __ cmp(r1, JVM_CONSTANT_Class);
3980   __ b(quicked, Assembler::EQ);
3981 
3982   __ push(atos); // save receiver for result, and for GC
3983   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3984   // vm_result_2 has metadata result
3985   __ get_vm_result_2(r0, rthread);
3986   __ pop(r3); // restore receiver
3987   __ b(resolved);
3988 
3989   // Get superklass in r0 and subklass in r3
3990   __ bind(quicked);
3991   __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
3992   __ lea(r0, Address(r2, r14, lsl(2)));
3993   __ ldr(r0, Address(r0, sizeof(ConstantPool)));
3994 
3995   __ bind(resolved);
3996   __ load_klass(r1, r3);
3997 
3998   // Generate subtype check.  Blows r2. Object in r3.
3999   // Superklass in r0. Subklass in r1.
4000   __ gen_subtype_check(r1, ok_is_subtype);
4001 
4002   // Come here on failure
4003   __ push(r3);
4004   // object is at TOS
4005   __ b(Interpreter::_throw_ClassCastException_entry);
4006 
4007   // Come here on success
4008   __ bind(ok_is_subtype);
4009   __ mov(r0, r3); // Restore object in r3
4010 
4011   // Collect counts on whether this test sees NULLs a lot or not.
4012   if (ProfileInterpreter) {
4013     __ b(done);
4014     __ bind(is_null);
4015     __ profile_null_seen(r2);
4016   } else {
4017     __ bind(is_null);   // same as 'done'
4018   }
4019   __ bind(done);
4020 }
4021 
4022 void TemplateTable::instanceof() {
4023   transition(atos, itos);
4024   Label done, is_null, ok_is_subtype, quicked, resolved;
4025   __ cbz(r0, is_null);
4026 
4027   // Get cpool & tags index
4028   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
4029   __ get_unsigned_2_byte_index_at_bcp(r14, 1); // r14=index
4030 
4031   // See if bytecode has already been quicked
4032   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
4033   __ ldrb(r1, Address(rscratch1, r14));
4034   __ cmp(r1, JVM_CONSTANT_Class);
4035   __ b(quicked, Assembler::EQ);
4036 
4037   __ push(atos); // save receiver for result, and for GC
4038   __ push_i(r14); // save index (used if profiling)
4039   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
4040   // vm_result_2 has metadata result
4041   __ get_vm_result_2(r0, rthread);
4042   __ pop_i(r14); // restore index
4043   __ pop(r3); // restore receiver
4044   __ verify_oop(r3);
4045   __ load_klass(r3, r3);
4046   __ b(resolved);
4047 
4048   // Get superklass in r0 and subklass in r3
4049   __ bind(quicked);
4050   __ load_klass(r3, r0);
4051   __ lea(r0, Address(r2, r14, lsl(2)));
4052   __ ldr(r0, Address(r0, sizeof(ConstantPool)));
4053 
4054   __ bind(resolved);
4055 
4056   // Generate subtype check.  Blows r2.
4057   // Superklass in r0.  Subklass in r3.
4058   __ gen_subtype_check(r3, ok_is_subtype);
4059 
4060   // Come here on failure
4061   __ mov(r0, 0);
4062   __ b(done);
4063   // Come here on success
4064   __ bind(ok_is_subtype);
4065   __ mov(r0, 1);
4066 
4067   // Collect counts on whether this test sees NULLs a lot or not.
4068   if (ProfileInterpreter) {
4069     __ b(done);
4070     __ bind(is_null);
4071     __ profile_null_seen(r2);
4072   } else {
4073     __ bind(is_null);   // same as 'done'
4074   }
4075   __ bind(done);
4076   // r0 = 0: obj == NULL or  obj is not an instanceof the specified klass
4077   // r0 = 1: obj != NULL and obj is     an instanceof the specified klass
4078 }
4079 
4080 //-----------------------------------------------------------------------------
4081 // Breakpoints
4082 void TemplateTable::_breakpoint() {
4083   // Note: We get here even if we are single stepping..
4084   // jbug inists on setting breakpoints at every bytecode
4085   // even if we are in single step mode.
4086 
4087   transition(vtos, vtos);
4088 
4089   // get the unpatched byte code
4090   __ get_method(c_rarg1);
4091   __ call_VM(noreg,
4092              CAST_FROM_FN_PTR(address,
4093                               InterpreterRuntime::get_original_bytecode_at),
4094              c_rarg1, rbcp);
4095   __ push(r0);
4096 
4097   // post the breakpoint event
4098   __ call_VM(noreg,
4099              CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
4100              rmethod, rbcp);
4101 
4102   // complete the execution of original bytecode
4103   __ pop(rscratch1);
4104   __ dispatch_only_normal(vtos);
4105 }
4106 
4107 //-----------------------------------------------------------------------------
4108 // Exceptions
4109 
4110 void TemplateTable::athrow() {
4111   transition(atos, vtos);
4112   __ null_check(r0);
4113   __ b(Interpreter::throw_exception_entry());
4114 }
4115 
4116 //-----------------------------------------------------------------------------
4117 // Synchronization
4118 //
4119 // Note: monitorenter & exit are symmetric routines; which is reflected
4120 //       in the assembly code structure as well
4121 //
4122 // Stack layout:
4123 //
4124 // [expressions  ] <--- sp                = expression stack top
4125 // ..
4126 // [expressions  ]
4127 // [monitor entry] <--- monitor block top = expression stack bot
4128 // ..
4129 // [monitor entry]
4130 // [frame data   ] <--- monitor block bot
4131 // ...
4132 // [saved rbp    ] <--- rbp
4133 void TemplateTable::monitorenter()
4134 {
4135   transition(atos, vtos);
4136 
4137   // check for NULL object
4138   __ null_check(r0);
4139 
4140   const Address monitor_block_top(
4141         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4142   const Address monitor_block_bot(
4143         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
4144   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4145 
4146   Label allocated;
4147 
4148   // initialize entry pointer
4149   __ mov(c_rarg1, 0); // points to free slot or NULL
4150 
4151   // find a free slot in the monitor block (result in c_rarg1)
4152   {
4153     Label entry, loop, exit;
4154     __ ldr(c_rarg3, monitor_block_top); // points to current entry,
4155                                         // starting with top-most entry
4156     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
4157 
4158     __ b(entry);
4159 
4160     __ bind(loop);
4161     // check if current entry is used
4162     // if not used then remember entry in c_rarg1
4163     __ ldr(rscratch1, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
4164     __ cmp(rscratch1, 0);
4165     __ mov(c_rarg1, c_rarg3, Assembler::EQ);
4166     // check if current entry is for same object
4167     __ cmp(r0, rscratch1);
4168     // if same object then stop searching
4169     __ b(exit, Assembler::EQ);
4170     // otherwise advance to next entry
4171     __ add(c_rarg3, c_rarg3, entry_size);
4172     __ bind(entry);
4173     // check if bottom reached
4174     __ cmp(c_rarg3, c_rarg2);
4175     // if not at bottom then check this entry
4176     __ b(loop, Assembler::NE);
4177     __ bind(exit);
4178   }
4179 
4180   __ cbnz(c_rarg1, allocated); // check if a slot has been found and
4181                             // if found, continue with that on
4182 
4183   // allocate one if there's no free slot
4184   {
4185     Label entry, loop; //, no_adjust;
4186     // 1. compute new pointers            // rsp: old expression stack top
4187     __ ldr(c_rarg1, monitor_block_bot);   // c_rarg1: old expression stack bottom
4188     __ sub(sp, sp, entry_size);           // move expression stack top
4189     __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
4190     __ mov(c_rarg3, sp);                  // set start value for copy loop
4191     __ str(c_rarg1, monitor_block_bot);   // set new monitor block bottom
4192 
4193     //__ cmp(sp, c_rarg3);                  // Check if we need to move sp
4194     //__ b(no_adjust, Assembler::LO);      // to allow more stack space
4195                                           // for our new sp
4196     //__ sub(sp, sp, 2 * wordSize);
4197     //__ bind(no_adjust);
4198 
4199     __ b(entry);
4200     // 2. move expression stack contents
4201     __ bind(loop);
4202     __ ldr(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
4203                                                    // word from old location
4204     __ str(c_rarg2, Address(c_rarg3, 0));          // and store it at new location
4205     __ add(c_rarg3, c_rarg3, wordSize);            // advance to next word
4206     __ bind(entry);
4207     __ cmp(c_rarg3, c_rarg1);        // check if bottom reached
4208     __ b(loop, Assembler::NE);      // if not at bottom then
4209                                      // copy next word
4210   }
4211 
4212   // call run-time routine
4213   // c_rarg1: points to monitor entry
4214   __ bind(allocated);
4215 
4216   // Increment bcp to point to the next bytecode, so exception
4217   // handling for async. exceptions work correctly.
4218   // The object has already been poped from the stack, so the
4219   // expression stack looks correct.
4220   __ add(rbcp, rbcp, 1); //inc
4221 
4222   // store object
4223   __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
4224   __ lock_object(c_rarg1);
4225 
4226   // check to make sure this monitor doesn't cause stack overflow after locking
4227   __ save_bcp();  // in case of exception
4228   __ generate_stack_overflow_check(0);
4229 
4230   // The bcp has already been incremented. Just need to dispatch to
4231   // next instruction.
4232   __ dispatch_next(vtos);
4233 }
4234 
4235 
4236 void TemplateTable::monitorexit()
4237 {
4238   transition(atos, vtos);
4239 
4240   // check for NULL object
4241   __ null_check(r0);
4242 
4243   const Address monitor_block_top(
4244         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4245   const Address monitor_block_bot(
4246         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
4247   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
4248 
4249   Label found;
4250 
4251   // find matching slot
4252   {
4253     Label entry, loop;
4254     __ ldr(c_rarg1, monitor_block_top); // points to current entry,
4255                                         // starting with top-most entry
4256     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
4257                                         // of monitor block
4258     __ b(entry);
4259 
4260     __ bind(loop);
4261     // check if current entry is for same object
4262     __ ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
4263     __ cmp(r0, rscratch1);
4264     // if same object then stop searching
4265     __ b(found, Assembler::EQ);
4266     // otherwise advance to next entry
4267     __ add(c_rarg1, c_rarg1, entry_size);
4268     __ bind(entry);
4269     // check if bottom reached
4270     __ cmp(c_rarg1, c_rarg2);
4271     // if not at bottom then check this entry
4272     __ b(loop, Assembler::NE);
4273   }
4274 
4275   // error handling. Unlocking was not block-structured
4276   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4277                    InterpreterRuntime::throw_illegal_monitor_state_exception));
4278   __ should_not_reach_here();
4279 
4280   // call run-time routine
4281   __ bind(found);
4282   __ push_ptr(r0); // make sure object is on stack (contract with oopMaps)
4283   __ unlock_object(c_rarg1);
4284   __ pop_ptr(r0); // discard object
4285 }
4286 
4287 
4288 // Wide instructions
4289 //J_UPDATE
4290 void TemplateTable::wide()
4291 {
4292   __ load_unsigned_byte(r14, at_bcp(1));
4293   __ mov(rscratch1, (address)Interpreter::_wentry_point);
4294   __ ldr(rscratch1, Address(rscratch1, r14, lsl(2)));
4295   __ b(rscratch1);
4296 }
4297 
4298 
4299 // Multi arrays
4300 //J_UPDATE
4301 void TemplateTable::multianewarray() {
4302   transition(vtos, atos);
4303   __ load_unsigned_byte(r0, at_bcp(3)); // get number of dimensions
4304   // last dim is on top of stack; we want address of first one:
4305   // first_addr = last_addr + (ndims - 1) * wordSize
4306   __ lea(c_rarg1, Address(sp, r0, lsl(2)));
4307   __ sub(c_rarg1, c_rarg1, wordSize);
4308   call_VM(r0,
4309           CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
4310           c_rarg1);
4311   __ load_unsigned_byte(r1, at_bcp(3));
4312   __ lea(sp, Address(sp, r1, lsl(2)));
4313 }
4314 #endif // !CC_INTERP