< prev index next >

src/hotspot/cpu/aarch64/templateTable_aarch64.cpp

Print this page

 155 static void do_oop_load(InterpreterMacroAssembler* _masm,
 156                         Address src,
 157                         Register dst,
 158                         DecoratorSet decorators) {
 159   __ load_heap_oop(dst, src, r10, r11, decorators);
 160 }
 161 
 162 Address TemplateTable::at_bcp(int offset) {
 163   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 164   return Address(rbcp, offset);
 165 }
 166 
 167 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 168                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 169                                    int byte_no)
 170 {
 171   if (!RewriteBytecodes)  return;
 172   Label L_patch_done;
 173 
 174   switch (bc) {

 175   case Bytecodes::_fast_aputfield:
 176   case Bytecodes::_fast_bputfield:
 177   case Bytecodes::_fast_zputfield:
 178   case Bytecodes::_fast_cputfield:
 179   case Bytecodes::_fast_dputfield:
 180   case Bytecodes::_fast_fputfield:
 181   case Bytecodes::_fast_iputfield:
 182   case Bytecodes::_fast_lputfield:
 183   case Bytecodes::_fast_sputfield:
 184     {
 185       // We skip bytecode quickening for putfield instructions when
 186       // the put_code written to the constant pool cache is zero.
 187       // This is required so that every execution of this instruction
 188       // calls out to InterpreterRuntime::resolve_get_put to do
 189       // additional, required work.
 190       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 191       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 192       __ load_field_entry(temp_reg, bc_reg);
 193       if (byte_no == f1_byte) {
 194         __ lea(temp_reg, Address(temp_reg, in_bytes(ResolvedFieldEntry::get_code_offset())));

 736   locals_index_wide(r1);
 737   __ ldr(r0, aaddress(r1));
 738 }
 739 
 740 void TemplateTable::index_check(Register array, Register index)
 741 {
 742   // destroys r1, rscratch1
 743   // sign extend index for use by indexed load
 744   // __ movl2ptr(index, index);
 745   // check index
 746   Register length = rscratch1;
 747   __ ldrw(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 748   __ cmpw(index, length);
 749   if (index != r1) {
 750     // ??? convention: move aberrant index into r1 for exception message
 751     assert(r1 != array, "different registers");
 752     __ mov(r1, index);
 753   }
 754   Label ok;
 755   __ br(Assembler::LO, ok);
 756     // ??? convention: move array into r3 for exception message
 757   __ mov(r3, array);
 758   __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 759   __ br(rscratch1);
 760   __ bind(ok);
 761 }
 762 
 763 void TemplateTable::iaload()
 764 {
 765   transition(itos, itos);
 766   __ mov(r1, r0);
 767   __ pop_ptr(r0);
 768   // r0: array
 769   // r1: index
 770   index_check(r0, r1); // leaves index in r1, kills rscratch1
 771   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
 772   __ access_load_at(T_INT, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(2)), noreg, noreg);
 773 }
 774 
 775 void TemplateTable::laload()
 776 {
 777   transition(itos, ltos);
 778   __ mov(r1, r0);
 779   __ pop_ptr(r0);

 799 void TemplateTable::daload()
 800 {
 801   transition(itos, dtos);
 802   __ mov(r1, r0);
 803   __ pop_ptr(r0);
 804   // r0: array
 805   // r1: index
 806   index_check(r0, r1); // leaves index in r1, kills rscratch1
 807   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
 808   __ access_load_at(T_DOUBLE, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(3)), noreg, noreg);
 809 }
 810 
 811 void TemplateTable::aaload()
 812 {
 813   transition(itos, atos);
 814   __ mov(r1, r0);
 815   __ pop_ptr(r0);
 816   // r0: array
 817   // r1: index
 818   index_check(r0, r1); // leaves index in r1, kills rscratch1
 819   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 820   do_oop_load(_masm,
 821               Address(r0, r1, Address::uxtw(LogBytesPerHeapOop)),
 822               r0,
 823               IS_ARRAY);















 824 }
 825 
 826 void TemplateTable::baload()
 827 {
 828   transition(itos, itos);
 829   __ mov(r1, r0);
 830   __ pop_ptr(r0);
 831   // r0: array
 832   // r1: index
 833   index_check(r0, r1); // leaves index in r1, kills rscratch1
 834   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
 835   __ access_load_at(T_BYTE, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(0)), noreg, noreg);
 836 }
 837 
 838 void TemplateTable::caload()
 839 {
 840   transition(itos, itos);
 841   __ mov(r1, r0);
 842   __ pop_ptr(r0);
 843   // r0: array

1090   // r1:  index
1091   // r3:  array
1092   index_check(r3, r1); // prefer index in r1
1093   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
1094   __ access_store_at(T_FLOAT, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(2)), noreg /* ftos */, noreg, noreg, noreg);
1095 }
1096 
1097 void TemplateTable::dastore() {
1098   transition(dtos, vtos);
1099   __ pop_i(r1);
1100   __ pop_ptr(r3);
1101   // v0: value
1102   // r1:  index
1103   // r3:  array
1104   index_check(r3, r1); // prefer index in r1
1105   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
1106   __ access_store_at(T_DOUBLE, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(3)), noreg /* dtos */, noreg, noreg, noreg);
1107 }
1108 
1109 void TemplateTable::aastore() {
1110   Label is_null, ok_is_subtype, done;
1111   transition(vtos, vtos);
1112   // stack: ..., array, index, value
1113   __ ldr(r0, at_tos());    // value
1114   __ ldr(r2, at_tos_p1()); // index
1115   __ ldr(r3, at_tos_p2()); // array
1116 
1117   Address element_address(r3, r4, Address::uxtw(LogBytesPerHeapOop));
1118 
1119   index_check(r3, r2);     // kills r1




1120   __ add(r4, r2, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);


1121 
1122   // do array store check - check for null value first
1123   __ cbz(r0, is_null);
1124 








1125   // Move subklass into r1
1126   __ load_klass(r1, r0);
1127   // Move superklass into r0
1128   __ load_klass(r0, r3);
1129   __ ldr(r0, Address(r0,
1130                      ObjArrayKlass::element_klass_offset()));
1131   // Compress array + index*oopSize + 12 into a single register.  Frees r2.
1132 
1133   // Generate subtype check.  Blows r2, r5
1134   // Superklass in r0.  Subklass in r1.
1135   __ gen_subtype_check(r1, ok_is_subtype);


1136 
1137   // Come here on failure
1138   // object is at TOS
1139   __ b(Interpreter::_throw_ArrayStoreException_entry);
1140 
1141   // Come here on success
1142   __ bind(ok_is_subtype);
1143 
1144   // Get the value we will store
1145   __ ldr(r0, at_tos());
1146   // Now store using the appropriate barrier
1147   do_oop_store(_masm, element_address, r0, IS_ARRAY);
1148   __ b(done);
1149 
1150   // Have a null in r0, r3=array, r2=index.  Store null at ary[idx]
1151   __ bind(is_null);
1152   __ profile_null_seen(r2);











1153 
1154   // Store a null
1155   do_oop_store(_masm, element_address, noreg, IS_ARRAY);









































1156 
1157   // Pop stack arguments
1158   __ bind(done);
1159   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1160 }
1161 
1162 void TemplateTable::bastore()
1163 {
1164   transition(itos, vtos);
1165   __ pop_i(r1);
1166   __ pop_ptr(r3);
1167   // r0: value
1168   // r1: index
1169   // r3: array
1170   index_check(r3, r1); // prefer index in r1
1171 
1172   // Need to check whether array is boolean or byte
1173   // since both types share the bastore bytecode.
1174   __ load_klass(r2, r3);
1175   __ ldrw(r2, Address(r2, Klass::layout_helper_offset()));

1940   __ br(j_not(cc), not_taken);
1941   branch(false, false);
1942   __ bind(not_taken);
1943   __ profile_not_taken_branch(r0);
1944 }
1945 
1946 void TemplateTable::if_nullcmp(Condition cc)
1947 {
1948   transition(atos, vtos);
1949   // assume branch is more often taken than not (loops use backward branches)
1950   Label not_taken;
1951   if (cc == equal)
1952     __ cbnz(r0, not_taken);
1953   else
1954     __ cbz(r0, not_taken);
1955   branch(false, false);
1956   __ bind(not_taken);
1957   __ profile_not_taken_branch(r0);
1958 }
1959 
1960 void TemplateTable::if_acmp(Condition cc)
1961 {
1962   transition(atos, vtos);
1963   // assume branch is more often taken than not (loops use backward branches)
1964   Label not_taken;
1965   __ pop_ptr(r1);






































1966   __ cmpoop(r1, r0);
1967   __ br(j_not(cc), not_taken);

1968   branch(false, false);
1969   __ bind(not_taken);
1970   __ profile_not_taken_branch(r0);









1971 }
1972 

1973 void TemplateTable::ret() {
1974   transition(vtos, vtos);
1975   locals_index(r1);
1976   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
1977   __ profile_ret(r1, r2);
1978   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
1979   __ lea(rbcp, Address(rbcp, r1));
1980   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
1981   __ dispatch_next(vtos, 0, /*generate_poll*/true);
1982 }
1983 
1984 void TemplateTable::wide_ret() {
1985   transition(vtos, vtos);
1986   locals_index_wide(r1);
1987   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
1988   __ profile_ret(r1, r2);
1989   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
1990   __ lea(rbcp, Address(rbcp, r1));
1991   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
1992   __ dispatch_next(vtos, 0, /*generate_poll*/true);

2562     }
2563     // c_rarg1: object pointer or null
2564     // c_rarg2: cache entry pointer
2565     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2566                                        InterpreterRuntime::post_field_access),
2567                c_rarg1, c_rarg2);
2568     __ load_field_entry(cache, index);
2569     __ bind(L1);
2570   }
2571 }
2572 
2573 void TemplateTable::pop_and_check_object(Register r)
2574 {
2575   __ pop_ptr(r);
2576   __ null_check(r);  // for field access must check obj.
2577   __ verify_oop(r);
2578 }
2579 
2580 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc)
2581 {
2582   const Register cache     = r4;
2583   const Register obj       = r4;



2584   const Register index     = r3;
2585   const Register tos_state = r3;
2586   const Register off       = r19;
2587   const Register flags     = r6;
2588   const Register bc        = r4; // uses same reg as obj, so don't mix them
2589 
2590   resolve_cache_and_index_for_field(byte_no, cache, index);
2591   jvmti_post_field_access(cache, index, is_static, false);





2592   load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
2593 
2594   if (!is_static) {
2595     // obj is on the stack
2596     pop_and_check_object(obj);
2597   }
2598 
2599   // 8179954: We need to make sure that the code generated for
2600   // volatile accesses forms a sequentially-consistent set of
2601   // operations when combined with STLR and LDAR.  Without a leading
2602   // membar it's possible for a simple Dekker test to fail if loads
2603   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
2604   // the stores in one method and we interpret the loads in another.
2605   if (!CompilerConfig::is_c1_or_interpreter_only_no_jvmci()){
2606     Label notVolatile;
2607     __ tbz(flags, ResolvedFieldEntry::is_volatile_shift, notVolatile);
2608     __ membar(MacroAssembler::AnyAny);
2609     __ bind(notVolatile);
2610   }
2611 

2630   __ b(Done);
2631 
2632   __ bind(notByte);
2633   __ cmp(tos_state, (u1)ztos);
2634   __ br(Assembler::NE, notBool);
2635 
2636   // ztos (same code as btos)
2637   __ access_load_at(T_BOOLEAN, IN_HEAP, r0, field, noreg, noreg);
2638   __ push(ztos);
2639   // Rewrite bytecode to be faster
2640   if (rc == may_rewrite) {
2641     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2642     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2643   }
2644   __ b(Done);
2645 
2646   __ bind(notBool);
2647   __ cmp(tos_state, (u1)atos);
2648   __ br(Assembler::NE, notObj);
2649   // atos
2650   do_oop_load(_masm, field, r0, IN_HEAP);
2651   __ push(atos);
2652   if (rc == may_rewrite) {
2653     patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);





































































2654   }
2655   __ b(Done);
2656 
2657   __ bind(notObj);
2658   __ cmp(tos_state, (u1)itos);
2659   __ br(Assembler::NE, notInt);
2660   // itos
2661   __ access_load_at(T_INT, IN_HEAP, r0, field, noreg, noreg);
2662   __ push(itos);
2663   // Rewrite bytecode to be faster
2664   if (rc == may_rewrite) {
2665     patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2666   }
2667   __ b(Done);
2668 
2669   __ bind(notInt);
2670   __ cmp(tos_state, (u1)ctos);
2671   __ br(Assembler::NE, notChar);
2672   // ctos
2673   __ access_load_at(T_CHAR, IN_HEAP, r0, field, noreg, noreg);
2674   __ push(ctos);
2675   // Rewrite bytecode to be faster

2796     // c_rarg1: object pointer set up above (null if static)
2797     // c_rarg2: cache entry pointer
2798     // c_rarg3: jvalue object on the stack
2799     __ call_VM(noreg,
2800                CAST_FROM_FN_PTR(address,
2801                                 InterpreterRuntime::post_field_modification),
2802                c_rarg1, c_rarg2, c_rarg3);
2803     __ load_field_entry(cache, index);
2804     __ bind(L1);
2805   }
2806 }
2807 
2808 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2809   transition(vtos, vtos);
2810 
2811   const Register cache     = r2;
2812   const Register index     = r3;
2813   const Register tos_state = r3;
2814   const Register obj       = r2;
2815   const Register off       = r19;
2816   const Register flags     = r0;
2817   const Register bc        = r4;

2818 
2819   resolve_cache_and_index_for_field(byte_no, cache, index);
2820   jvmti_post_field_mod(cache, index, is_static);
2821   load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
2822 
2823   Label Done;
2824   __ mov(r5, flags);
2825 
2826   {
2827     Label notVolatile;
2828     __ tbz(r5, ResolvedFieldEntry::is_volatile_shift, notVolatile);
2829     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
2830     __ bind(notVolatile);
2831   }
2832 
2833   // field address
2834   const Address field(obj, off);
2835 
2836   Label notByte, notBool, notInt, notShort, notChar,
2837         notLong, notFloat, notObj, notDouble;
2838 
2839   assert(btos == 0, "change code, btos != 0");
2840   __ cbnz(tos_state, notByte);
2841 
2842   // Don't rewrite putstatic, only putfield
2843   if (is_static) rc = may_not_rewrite;
2844 
2845   // btos
2846   {
2847     __ pop(btos);
2848     if (!is_static) pop_and_check_object(obj);

2857   __ cmp(tos_state, (u1)ztos);
2858   __ br(Assembler::NE, notBool);
2859 
2860   // ztos
2861   {
2862     __ pop(ztos);
2863     if (!is_static) pop_and_check_object(obj);
2864     __ access_store_at(T_BOOLEAN, IN_HEAP, field, r0, noreg, noreg, noreg);
2865     if (rc == may_rewrite) {
2866       patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
2867     }
2868     __ b(Done);
2869   }
2870 
2871   __ bind(notBool);
2872   __ cmp(tos_state, (u1)atos);
2873   __ br(Assembler::NE, notObj);
2874 
2875   // atos
2876   {
2877     __ pop(atos);
2878     if (!is_static) pop_and_check_object(obj);
2879     // Store into the field
2880     do_oop_store(_masm, field, r0, IN_HEAP);
2881     if (rc == may_rewrite) {
2882       patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
2883     }
2884     __ b(Done);




















































2885   }
2886 
2887   __ bind(notObj);
2888   __ cmp(tos_state, (u1)itos);
2889   __ br(Assembler::NE, notInt);
2890 
2891   // itos
2892   {
2893     __ pop(itos);
2894     if (!is_static) pop_and_check_object(obj);
2895     __ access_store_at(T_INT, IN_HEAP, field, r0, noreg, noreg, noreg);
2896     if (rc == may_rewrite) {
2897       patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
2898     }
2899     __ b(Done);
2900   }
2901 
2902   __ bind(notInt);
2903   __ cmp(tos_state, (u1)ctos);
2904   __ br(Assembler::NE, notChar);

2969   {
2970     __ pop(dtos);
2971     if (!is_static) pop_and_check_object(obj);
2972     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg, noreg);
2973     if (rc == may_rewrite) {
2974       patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
2975     }
2976   }
2977 
2978 #ifdef ASSERT
2979   __ b(Done);
2980 
2981   __ bind(notDouble);
2982   __ stop("Bad state");
2983 #endif
2984 
2985   __ bind(Done);
2986 
2987   {
2988     Label notVolatile;
2989     __ tbz(r5, ResolvedFieldEntry::is_volatile_shift, notVolatile);
2990     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
2991     __ bind(notVolatile);
2992   }
2993 }
2994 
2995 void TemplateTable::putfield(int byte_no)
2996 {
2997   putfield_or_static(byte_no, false);
2998 }
2999 
3000 void TemplateTable::nofast_putfield(int byte_no) {
3001   putfield_or_static(byte_no, false, may_not_rewrite);
3002 }
3003 
3004 void TemplateTable::putstatic(int byte_no) {
3005   putfield_or_static(byte_no, true);
3006 }
3007 
3008 void TemplateTable::jvmti_post_fast_field_mod() {
3009   if (JvmtiExport::can_post_field_modification()) {
3010     // Check to see if a field modification watch has been set before
3011     // we take the time to call into the VM.
3012     Label L2;
3013     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
3014     __ ldrw(c_rarg3, Address(rscratch1));
3015     __ cbzw(c_rarg3, L2);
3016     __ pop_ptr(r19);                  // copy the object pointer from tos
3017     __ verify_oop(r19);
3018     __ push_ptr(r19);                 // put the object pointer back on tos
3019     // Save tos values before call_VM() clobbers them. Since we have
3020     // to do it for every data type, we use the saved values as the
3021     // jvalue object.
3022     switch (bytecode()) {          // load values into the jvalue object

3023     case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
3024     case Bytecodes::_fast_bputfield: // fall through
3025     case Bytecodes::_fast_zputfield: // fall through
3026     case Bytecodes::_fast_sputfield: // fall through
3027     case Bytecodes::_fast_cputfield: // fall through
3028     case Bytecodes::_fast_iputfield: __ push_i(r0); break;
3029     case Bytecodes::_fast_dputfield: __ push_d(); break;
3030     case Bytecodes::_fast_fputfield: __ push_f(); break;
3031     case Bytecodes::_fast_lputfield: __ push_l(r0); break;
3032 
3033     default:
3034       ShouldNotReachHere();
3035     }
3036     __ mov(c_rarg3, esp);             // points to jvalue on the stack
3037     // access constant pool cache entry
3038     __ load_field_entry(c_rarg2, r0);
3039     __ verify_oop(r19);
3040     // r19: object pointer copied above
3041     // c_rarg2: cache entry pointer
3042     // c_rarg3: jvalue object on the stack
3043     __ call_VM(noreg,
3044                CAST_FROM_FN_PTR(address,
3045                                 InterpreterRuntime::post_field_modification),
3046                r19, c_rarg2, c_rarg3);
3047 
3048     switch (bytecode()) {             // restore tos values

3049     case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
3050     case Bytecodes::_fast_bputfield: // fall through
3051     case Bytecodes::_fast_zputfield: // fall through
3052     case Bytecodes::_fast_sputfield: // fall through
3053     case Bytecodes::_fast_cputfield: // fall through
3054     case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
3055     case Bytecodes::_fast_dputfield: __ pop_d(); break;
3056     case Bytecodes::_fast_fputfield: __ pop_f(); break;
3057     case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
3058     default: break;
3059     }
3060     __ bind(L2);
3061   }
3062 }
3063 
3064 void TemplateTable::fast_storefield(TosState state)
3065 {
3066   transition(state, vtos);
3067 
3068   ByteSize base = ConstantPoolCache::base_offset();

3075   // R1: field offset, R2: field holder, R3: flags
3076   load_resolved_field_entry(r2, r2, noreg, r1, r3);
3077 
3078   {
3079     Label notVolatile;
3080     __ tbz(r3, ResolvedFieldEntry::is_volatile_shift, notVolatile);
3081     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
3082     __ bind(notVolatile);
3083   }
3084 
3085   Label notVolatile;
3086 
3087   // Get object from stack
3088   pop_and_check_object(r2);
3089 
3090   // field address
3091   const Address field(r2, r1);
3092 
3093   // access field
3094   switch (bytecode()) {























3095   case Bytecodes::_fast_aputfield:
3096     do_oop_store(_masm, field, r0, IN_HEAP);
3097     break;
3098   case Bytecodes::_fast_lputfield:
3099     __ access_store_at(T_LONG, IN_HEAP, field, r0, noreg, noreg, noreg);
3100     break;
3101   case Bytecodes::_fast_iputfield:
3102     __ access_store_at(T_INT, IN_HEAP, field, r0, noreg, noreg, noreg);
3103     break;
3104   case Bytecodes::_fast_zputfield:
3105     __ access_store_at(T_BOOLEAN, IN_HEAP, field, r0, noreg, noreg, noreg);
3106     break;
3107   case Bytecodes::_fast_bputfield:
3108     __ access_store_at(T_BYTE, IN_HEAP, field, r0, noreg, noreg, noreg);
3109     break;
3110   case Bytecodes::_fast_sputfield:
3111     __ access_store_at(T_SHORT, IN_HEAP, field, r0, noreg, noreg, noreg);
3112     break;
3113   case Bytecodes::_fast_cputfield:
3114     __ access_store_at(T_CHAR, IN_HEAP, field, r0, noreg, noreg, noreg);

3167   // r0: object
3168   __ verify_oop(r0);
3169   __ null_check(r0);
3170   const Address field(r0, r1);
3171 
3172   // 8179954: We need to make sure that the code generated for
3173   // volatile accesses forms a sequentially-consistent set of
3174   // operations when combined with STLR and LDAR.  Without a leading
3175   // membar it's possible for a simple Dekker test to fail if loads
3176   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3177   // the stores in one method and we interpret the loads in another.
3178   if (!CompilerConfig::is_c1_or_interpreter_only_no_jvmci()) {
3179     Label notVolatile;
3180     __ tbz(r3, ResolvedFieldEntry::is_volatile_shift, notVolatile);
3181     __ membar(MacroAssembler::AnyAny);
3182     __ bind(notVolatile);
3183   }
3184 
3185   // access field
3186   switch (bytecode()) {





























3187   case Bytecodes::_fast_agetfield:
3188     do_oop_load(_masm, field, r0, IN_HEAP);
3189     __ verify_oop(r0);
3190     break;
3191   case Bytecodes::_fast_lgetfield:
3192     __ access_load_at(T_LONG, IN_HEAP, r0, field, noreg, noreg);
3193     break;
3194   case Bytecodes::_fast_igetfield:
3195     __ access_load_at(T_INT, IN_HEAP, r0, field, noreg, noreg);
3196     break;
3197   case Bytecodes::_fast_bgetfield:
3198     __ access_load_at(T_BYTE, IN_HEAP, r0, field, noreg, noreg);
3199     break;
3200   case Bytecodes::_fast_sgetfield:
3201     __ access_load_at(T_SHORT, IN_HEAP, r0, field, noreg, noreg);
3202     break;
3203   case Bytecodes::_fast_cgetfield:
3204     __ access_load_at(T_CHAR, IN_HEAP, r0, field, noreg, noreg);
3205     break;
3206   case Bytecodes::_fast_fgetfield:

3585   Label initialize_header;
3586 
3587   __ get_cpool_and_tags(r4, r0);
3588   // Make sure the class we're about to instantiate has been resolved.
3589   // This is done before loading InstanceKlass to be consistent with the order
3590   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3591   const int tags_offset = Array<u1>::base_offset_in_bytes();
3592   __ lea(rscratch1, Address(r0, r3, Address::lsl(0)));
3593   __ lea(rscratch1, Address(rscratch1, tags_offset));
3594   __ ldarb(rscratch1, rscratch1);
3595   __ cmp(rscratch1, (u1)JVM_CONSTANT_Class);
3596   __ br(Assembler::NE, slow_case);
3597 
3598   // get InstanceKlass
3599   __ load_resolved_klass_at_offset(r4, r3, r4, rscratch1);
3600 
3601   // make sure klass is initialized
3602   assert(VM_Version::supports_fast_class_init_checks(), "Optimization requires support for fast class initialization checks");
3603   __ clinit_barrier(r4, rscratch1, nullptr /*L_fast_path*/, &slow_case);
3604 
3605   // get instance_size in InstanceKlass (scaled to a count of bytes)
3606   __ ldrw(r3,
3607           Address(r4,
3608                   Klass::layout_helper_offset()));
3609   // test to see if it is malformed in some way
3610   __ tbnz(r3, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
3611 
3612   // Allocate the instance:
3613   //  If TLAB is enabled:
3614   //    Try to allocate in the TLAB.
3615   //    If fails, go to the slow path.
3616   //    Initialize the allocation.
3617   //    Exit.
3618   //
3619   //  Go to slow path.
3620 
3621   if (UseTLAB) {
3622     __ tlab_allocate(r0, r3, 0, noreg, r1, slow_case);
3623 
3624     if (ZeroTLAB) {
3625       // the fields have been already cleared
3626       __ b(initialize_header);
3627     }
3628 
3629     // The object is initialized before the header.  If the object size is
3630     // zero, go directly to the header initialization.
3631     __ sub(r3, r3, sizeof(oopDesc));
3632     __ cbz(r3, initialize_header);
3633 
3634     // Initialize object fields
3635     {
3636       __ add(r2, r0, sizeof(oopDesc));
3637       Label loop;
3638       __ bind(loop);
3639       __ str(zr, Address(__ post(r2, BytesPerLong)));
3640       __ sub(r3, r3, BytesPerLong);
3641       __ cbnz(r3, loop);
3642     }
3643 
3644     // initialize object header only.
3645     __ bind(initialize_header);
3646     __ mov(rscratch1, (intptr_t)markWord::prototype().value());
3647     __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
3648     __ store_klass_gap(r0, zr);  // zero klass gap for compressed oops
3649     __ store_klass(r0, r4);      // store klass last
3650 
3651     {
3652       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3653       // Trigger dtrace event for fastpath
3654       __ push(atos); // save the return value
3655       __ call_VM_leaf(
3656            CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), r0);
3657       __ pop(atos); // restore the return value
3658 
3659     }
3660     __ b(done);
3661   }
3662 
3663   // slow case
3664   __ bind(slow_case);
3665   __ get_constant_pool(c_rarg1);
3666   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3667   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3668   __ verify_oop(r0);
3669 
3670   // continue
3671   __ bind(done);
3672   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3673   __ membar(Assembler::StoreStore);
3674 }
3675 
3676 void TemplateTable::newarray() {
3677   transition(itos, atos);
3678   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3679   __ mov(c_rarg2, r0);
3680   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3681           c_rarg1, c_rarg2);

3726   __ bind(quicked);
3727   __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
3728   __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1); // r0 = klass
3729 
3730   __ bind(resolved);
3731   __ load_klass(r19, r3);
3732 
3733   // Generate subtype check.  Blows r2, r5.  Object in r3.
3734   // Superklass in r0.  Subklass in r19.
3735   __ gen_subtype_check(r19, ok_is_subtype);
3736 
3737   // Come here on failure
3738   __ push(r3);
3739   // object is at TOS
3740   __ b(Interpreter::_throw_ClassCastException_entry);
3741 
3742   // Come here on success
3743   __ bind(ok_is_subtype);
3744   __ mov(r0, r3); // Restore object in r3
3745 



3746   // Collect counts on whether this test sees nulls a lot or not.
3747   if (ProfileInterpreter) {
3748     __ b(done);
3749     __ bind(is_null);
3750     __ profile_null_seen(r2);
3751   } else {
3752     __ bind(is_null);   // same as 'done'
3753   }

3754   __ bind(done);
3755 }
3756 
3757 void TemplateTable::instanceof() {
3758   transition(atos, itos);
3759   Label done, is_null, ok_is_subtype, quicked, resolved;
3760   __ cbz(r0, is_null);
3761 
3762   // Get cpool & tags index
3763   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3764   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3765   // See if bytecode has already been quicked
3766   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3767   __ lea(r1, Address(rscratch1, r19));
3768   __ ldarb(r1, r1);
3769   __ cmp(r1, (u1)JVM_CONSTANT_Class);
3770   __ br(Assembler::EQ, quicked);
3771 
3772   __ push(atos); // save receiver for result, and for GC
3773   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));

3852 //       in the assembly code structure as well
3853 //
3854 // Stack layout:
3855 //
3856 // [expressions  ] <--- esp               = expression stack top
3857 // ..
3858 // [expressions  ]
3859 // [monitor entry] <--- monitor block top = expression stack bot
3860 // ..
3861 // [monitor entry]
3862 // [frame data   ] <--- monitor block bot
3863 // ...
3864 // [saved rfp    ] <--- rfp
3865 void TemplateTable::monitorenter()
3866 {
3867   transition(atos, vtos);
3868 
3869   // check for null object
3870   __ null_check(r0);
3871 




3872   const Address monitor_block_top(
3873         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3874   const Address monitor_block_bot(
3875         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3876   const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
3877 
3878   Label allocated;
3879 
3880   // initialize entry pointer
3881   __ mov(c_rarg1, zr); // points to free slot or null
3882 
3883   // find a free slot in the monitor block (result in c_rarg1)
3884   {
3885     Label entry, loop, exit;
3886     __ ldr(c_rarg3, monitor_block_top); // derelativize pointer
3887     __ lea(c_rarg3, Address(rfp, c_rarg3, Address::lsl(Interpreter::logStackElementSize)));
3888     // c_rarg3 points to current entry, starting with top-most entry
3889 
3890     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3891 

3953   // c_rarg1: points to monitor entry
3954   __ bind(allocated);
3955 
3956   // Increment bcp to point to the next bytecode, so exception
3957   // handling for async. exceptions work correctly.
3958   // The object has already been popped from the stack, so the
3959   // expression stack looks correct.
3960   __ increment(rbcp);
3961 
3962   // store object
3963   __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset()));
3964   __ lock_object(c_rarg1);
3965 
3966   // check to make sure this monitor doesn't cause stack overflow after locking
3967   __ save_bcp();  // in case of exception
3968   __ generate_stack_overflow_check(0);
3969 
3970   // The bcp has already been incremented. Just need to dispatch to
3971   // next instruction.
3972   __ dispatch_next(vtos);





3973 }
3974 
3975 
3976 void TemplateTable::monitorexit()
3977 {
3978   transition(atos, vtos);
3979 
3980   // check for null object
3981   __ null_check(r0);
3982 












3983   const Address monitor_block_top(
3984         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3985   const Address monitor_block_bot(
3986         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3987   const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
3988 
3989   Label found;
3990 
3991   // find matching slot
3992   {
3993     Label entry, loop;
3994     __ ldr(c_rarg1, monitor_block_top); // derelativize pointer
3995     __ lea(c_rarg1, Address(rfp, c_rarg1, Address::lsl(Interpreter::logStackElementSize)));
3996     // c_rarg1 points to current entry, starting with top-most entry
3997 
3998     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3999                                         // of monitor block
4000     __ b(entry);
4001 
4002     __ bind(loop);

 155 static void do_oop_load(InterpreterMacroAssembler* _masm,
 156                         Address src,
 157                         Register dst,
 158                         DecoratorSet decorators) {
 159   __ load_heap_oop(dst, src, r10, r11, decorators);
 160 }
 161 
 162 Address TemplateTable::at_bcp(int offset) {
 163   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 164   return Address(rbcp, offset);
 165 }
 166 
 167 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 168                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 169                                    int byte_no)
 170 {
 171   if (!RewriteBytecodes)  return;
 172   Label L_patch_done;
 173 
 174   switch (bc) {
 175   case Bytecodes::_fast_vputfield:
 176   case Bytecodes::_fast_aputfield:
 177   case Bytecodes::_fast_bputfield:
 178   case Bytecodes::_fast_zputfield:
 179   case Bytecodes::_fast_cputfield:
 180   case Bytecodes::_fast_dputfield:
 181   case Bytecodes::_fast_fputfield:
 182   case Bytecodes::_fast_iputfield:
 183   case Bytecodes::_fast_lputfield:
 184   case Bytecodes::_fast_sputfield:
 185     {
 186       // We skip bytecode quickening for putfield instructions when
 187       // the put_code written to the constant pool cache is zero.
 188       // This is required so that every execution of this instruction
 189       // calls out to InterpreterRuntime::resolve_get_put to do
 190       // additional, required work.
 191       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 192       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 193       __ load_field_entry(temp_reg, bc_reg);
 194       if (byte_no == f1_byte) {
 195         __ lea(temp_reg, Address(temp_reg, in_bytes(ResolvedFieldEntry::get_code_offset())));

 737   locals_index_wide(r1);
 738   __ ldr(r0, aaddress(r1));
 739 }
 740 
 741 void TemplateTable::index_check(Register array, Register index)
 742 {
 743   // destroys r1, rscratch1
 744   // sign extend index for use by indexed load
 745   // __ movl2ptr(index, index);
 746   // check index
 747   Register length = rscratch1;
 748   __ ldrw(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
 749   __ cmpw(index, length);
 750   if (index != r1) {
 751     // ??? convention: move aberrant index into r1 for exception message
 752     assert(r1 != array, "different registers");
 753     __ mov(r1, index);
 754   }
 755   Label ok;
 756   __ br(Assembler::LO, ok);
 757   // ??? convention: move array into r3 for exception message
 758    __ mov(r3, array);
 759    __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
 760    __ br(rscratch1);
 761   __ bind(ok);
 762 }
 763 
 764 void TemplateTable::iaload()
 765 {
 766   transition(itos, itos);
 767   __ mov(r1, r0);
 768   __ pop_ptr(r0);
 769   // r0: array
 770   // r1: index
 771   index_check(r0, r1); // leaves index in r1, kills rscratch1
 772   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2);
 773   __ access_load_at(T_INT, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(2)), noreg, noreg);
 774 }
 775 
 776 void TemplateTable::laload()
 777 {
 778   transition(itos, ltos);
 779   __ mov(r1, r0);
 780   __ pop_ptr(r0);

 800 void TemplateTable::daload()
 801 {
 802   transition(itos, dtos);
 803   __ mov(r1, r0);
 804   __ pop_ptr(r0);
 805   // r0: array
 806   // r1: index
 807   index_check(r0, r1); // leaves index in r1, kills rscratch1
 808   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
 809   __ access_load_at(T_DOUBLE, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(3)), noreg, noreg);
 810 }
 811 
 812 void TemplateTable::aaload()
 813 {
 814   transition(itos, atos);
 815   __ mov(r1, r0);
 816   __ pop_ptr(r0);
 817   // r0: array
 818   // r1: index
 819   index_check(r0, r1); // leaves index in r1, kills rscratch1
 820   __ profile_array_type<ArrayLoadData>(r2, r0, r4);
 821   if (UseFlatArray) {
 822     Label is_flat_array, done;
 823 
 824     __ test_flat_array_oop(r0, r8 /*temp*/, is_flat_array);
 825     __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 826     do_oop_load(_masm, Address(r0, r1, Address::uxtw(LogBytesPerHeapOop)), r0, IS_ARRAY);
 827 
 828     __ b(done);
 829     __ bind(is_flat_array);
 830     __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_load), r0, r1);
 831     // Ensure the stores to copy the inline field contents are visible
 832     // before any subsequent store that publishes this reference.
 833     __ membar(Assembler::StoreStore);
 834     __ bind(done);
 835   } else {
 836     __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
 837     do_oop_load(_masm, Address(r0, r1, Address::uxtw(LogBytesPerHeapOop)), r0, IS_ARRAY);
 838   }
 839   __ profile_element_type(r2, r0, r4);
 840 }
 841 
 842 void TemplateTable::baload()
 843 {
 844   transition(itos, itos);
 845   __ mov(r1, r0);
 846   __ pop_ptr(r0);
 847   // r0: array
 848   // r1: index
 849   index_check(r0, r1); // leaves index in r1, kills rscratch1
 850   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0);
 851   __ access_load_at(T_BYTE, IN_HEAP | IS_ARRAY, r0, Address(r0, r1, Address::uxtw(0)), noreg, noreg);
 852 }
 853 
 854 void TemplateTable::caload()
 855 {
 856   transition(itos, itos);
 857   __ mov(r1, r0);
 858   __ pop_ptr(r0);
 859   // r0: array

1106   // r1:  index
1107   // r3:  array
1108   index_check(r3, r1); // prefer index in r1
1109   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2);
1110   __ access_store_at(T_FLOAT, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(2)), noreg /* ftos */, noreg, noreg, noreg);
1111 }
1112 
1113 void TemplateTable::dastore() {
1114   transition(dtos, vtos);
1115   __ pop_i(r1);
1116   __ pop_ptr(r3);
1117   // v0: value
1118   // r1:  index
1119   // r3:  array
1120   index_check(r3, r1); // prefer index in r1
1121   __ add(r1, r1, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3);
1122   __ access_store_at(T_DOUBLE, IN_HEAP | IS_ARRAY, Address(r3, r1, Address::uxtw(3)), noreg /* dtos */, noreg, noreg, noreg);
1123 }
1124 
1125 void TemplateTable::aastore() {
1126   Label is_null, is_flat_array, ok_is_subtype, done;
1127   transition(vtos, vtos);
1128   // stack: ..., array, index, value
1129   __ ldr(r0, at_tos());    // value
1130   __ ldr(r2, at_tos_p1()); // index
1131   __ ldr(r3, at_tos_p2()); // array
1132 


1133   index_check(r3, r2);     // kills r1
1134 
1135   __ profile_array_type<ArrayStoreData>(r4, r3, r5);
1136   __ profile_multiple_element_types(r4, r0, r5, r6);
1137 
1138   __ add(r4, r2, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
1139   Address element_address(r3, r4, Address::uxtw(LogBytesPerHeapOop));
1140   // Be careful not to clobber r4 below
1141 
1142   // do array store check - check for null value first
1143   __ cbz(r0, is_null);
1144 
1145   // Move array class to r5
1146   __ load_klass(r5, r3);
1147 
1148   if (UseFlatArray) {
1149     __ ldrw(r6, Address(r5, Klass::layout_helper_offset()));
1150     __ test_flat_array_layout(r6, is_flat_array);
1151   }
1152 
1153   // Move subklass into r1
1154   __ load_klass(r1, r0);
1155 
1156   // Move array element superklass into r0
1157   __ ldr(r0, Address(r5, ObjArrayKlass::element_klass_offset()));

1158   // Compress array + index*oopSize + 12 into a single register.  Frees r2.
1159 
1160   // Generate subtype check.  Blows r2, r5
1161   // Superklass in r0.  Subklass in r1.
1162 
1163   // is "r1 <: r0" ? (value subclass <: array element superclass)
1164   __ gen_subtype_check(r1, ok_is_subtype, false);
1165 
1166   // Come here on failure
1167   // object is at TOS
1168   __ b(Interpreter::_throw_ArrayStoreException_entry);
1169 
1170   // Come here on success
1171   __ bind(ok_is_subtype);
1172 
1173   // Get the value we will store
1174   __ ldr(r0, at_tos());
1175   // Now store using the appropriate barrier
1176   do_oop_store(_masm, element_address, r0, IS_ARRAY);
1177   __ b(done);
1178 
1179   // Have a null in r0, r3=array, r2=index.  Store null at ary[idx]
1180   __ bind(is_null);
1181   if (EnableValhalla) {
1182     Label is_null_into_value_array_npe, store_null;
1183 
1184     // No way to store null in flat null-free array
1185     __ test_null_free_array_oop(r3, r8, is_null_into_value_array_npe);
1186     __ b(store_null);
1187 
1188     __ bind(is_null_into_value_array_npe);
1189     __ b(ExternalAddress(Interpreter::_throw_NullPointerException_entry));
1190 
1191     __ bind(store_null);
1192   }
1193 
1194   // Store a null
1195   do_oop_store(_masm, element_address, noreg, IS_ARRAY);
1196   __ b(done);
1197 
1198   if (UseFlatArray) {
1199      Label is_type_ok;
1200     __ bind(is_flat_array); // Store non-null value to flat
1201 
1202     // Simplistic type check...
1203     // r0 - value, r2 - index, r3 - array.
1204 
1205     // Profile the not-null value's klass.
1206     // Load value class
1207      __ load_klass(r1, r0);
1208 
1209     // Move element klass into r7
1210      __ ldr(r7, Address(r5, ArrayKlass::element_klass_offset()));
1211 
1212     // flat value array needs exact type match
1213     // is "r1 == r7" (value subclass == array element superclass)
1214 
1215      __ cmp(r7, r1);
1216      __ br(Assembler::EQ, is_type_ok);
1217 
1218      __ b(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
1219 
1220      __ bind(is_type_ok);
1221     // r1: value's klass
1222     // r3: array
1223     // r5: array klass
1224     __ test_klass_is_empty_inline_type(r1, r7, done);
1225 
1226     // calc dst for copy
1227     __ ldrw(r7, at_tos_p1()); // index
1228     __ data_for_value_array_index(r3, r5, r7, r7);
1229 
1230     // ...and src for copy
1231     __ ldr(r6, at_tos());  // value
1232     __ data_for_oop(r6, r6, r1);
1233 
1234     __ mov(r4, r1);  // Shuffle arguments to avoid conflict with c_rarg1
1235     __ access_value_copy(IN_HEAP, r6, r7, r4);
1236   }
1237 
1238   // Pop stack arguments
1239   __ bind(done);
1240   __ add(esp, esp, 3 * Interpreter::stackElementSize);
1241 }
1242 
1243 void TemplateTable::bastore()
1244 {
1245   transition(itos, vtos);
1246   __ pop_i(r1);
1247   __ pop_ptr(r3);
1248   // r0: value
1249   // r1: index
1250   // r3: array
1251   index_check(r3, r1); // prefer index in r1
1252 
1253   // Need to check whether array is boolean or byte
1254   // since both types share the bastore bytecode.
1255   __ load_klass(r2, r3);
1256   __ ldrw(r2, Address(r2, Klass::layout_helper_offset()));

2021   __ br(j_not(cc), not_taken);
2022   branch(false, false);
2023   __ bind(not_taken);
2024   __ profile_not_taken_branch(r0);
2025 }
2026 
2027 void TemplateTable::if_nullcmp(Condition cc)
2028 {
2029   transition(atos, vtos);
2030   // assume branch is more often taken than not (loops use backward branches)
2031   Label not_taken;
2032   if (cc == equal)
2033     __ cbnz(r0, not_taken);
2034   else
2035     __ cbz(r0, not_taken);
2036   branch(false, false);
2037   __ bind(not_taken);
2038   __ profile_not_taken_branch(r0);
2039 }
2040 
2041 void TemplateTable::if_acmp(Condition cc) {

2042   transition(atos, vtos);
2043   // assume branch is more often taken than not (loops use backward branches)
2044   Label taken, not_taken;
2045   __ pop_ptr(r1);
2046 
2047   __ profile_acmp(r2, r1, r0, r4);
2048 
2049   Register is_inline_type_mask = rscratch1;
2050   __ mov(is_inline_type_mask, markWord::inline_type_pattern);
2051 
2052   if (EnableValhalla) {
2053     __ cmp(r1, r0);
2054     __ br(Assembler::EQ, (cc == equal) ? taken : not_taken);
2055 
2056     // might be substitutable, test if either r0 or r1 is null
2057     __ andr(r2, r0, r1);
2058     __ cbz(r2, (cc == equal) ? not_taken : taken);
2059 
2060     // and both are values ?
2061     __ ldr(r2, Address(r1, oopDesc::mark_offset_in_bytes()));
2062     __ andr(r2, r2, is_inline_type_mask);
2063     __ ldr(r4, Address(r0, oopDesc::mark_offset_in_bytes()));
2064     __ andr(r4, r4, is_inline_type_mask);
2065     __ andr(r2, r2, r4);
2066     __ cmp(r2,  is_inline_type_mask);
2067     __ br(Assembler::NE, (cc == equal) ? not_taken : taken);
2068 
2069     // same value klass ?
2070     __ load_metadata(r2, r1);
2071     __ load_metadata(r4, r0);
2072     __ cmp(r2, r4);
2073     __ br(Assembler::NE, (cc == equal) ? not_taken : taken);
2074 
2075     // Know both are the same type, let's test for substitutability...
2076     if (cc == equal) {
2077       invoke_is_substitutable(r0, r1, taken, not_taken);
2078     } else {
2079       invoke_is_substitutable(r0, r1, not_taken, taken);
2080     }
2081     __ stop("Not reachable");
2082   }
2083 
2084   __ cmpoop(r1, r0);
2085   __ br(j_not(cc), not_taken);
2086   __ bind(taken);
2087   branch(false, false);
2088   __ bind(not_taken);
2089   __ profile_not_taken_branch(r0, true);
2090 }
2091 
2092 void TemplateTable::invoke_is_substitutable(Register aobj, Register bobj,
2093                                             Label& is_subst, Label& not_subst) {
2094 
2095   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::is_substitutable), aobj, bobj);
2096   // Restored... r0 answer, jmp to outcome...
2097   __ cbz(r0, not_subst);
2098   __ b(is_subst);
2099 }
2100 
2101 
2102 void TemplateTable::ret() {
2103   transition(vtos, vtos);
2104   locals_index(r1);
2105   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2106   __ profile_ret(r1, r2);
2107   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2108   __ lea(rbcp, Address(rbcp, r1));
2109   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2110   __ dispatch_next(vtos, 0, /*generate_poll*/true);
2111 }
2112 
2113 void TemplateTable::wide_ret() {
2114   transition(vtos, vtos);
2115   locals_index_wide(r1);
2116   __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
2117   __ profile_ret(r1, r2);
2118   __ ldr(rbcp, Address(rmethod, Method::const_offset()));
2119   __ lea(rbcp, Address(rbcp, r1));
2120   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
2121   __ dispatch_next(vtos, 0, /*generate_poll*/true);

2691     }
2692     // c_rarg1: object pointer or null
2693     // c_rarg2: cache entry pointer
2694     __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2695                                        InterpreterRuntime::post_field_access),
2696                c_rarg1, c_rarg2);
2697     __ load_field_entry(cache, index);
2698     __ bind(L1);
2699   }
2700 }
2701 
2702 void TemplateTable::pop_and_check_object(Register r)
2703 {
2704   __ pop_ptr(r);
2705   __ null_check(r);  // for field access must check obj.
2706   __ verify_oop(r);
2707 }
2708 
2709 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc)
2710 {
2711   const Register cache     = r2;
2712   const Register obj       = r4;
2713   const Register klass     = r5;
2714   const Register inline_klass = r7;
2715   const Register field_index = r23;
2716   const Register index     = r3;
2717   const Register tos_state = r3;
2718   const Register off       = r19;
2719   const Register flags     = r6;
2720   const Register bc        = r4; // uses same reg as obj, so don't mix them
2721 
2722   resolve_cache_and_index_for_field(byte_no, cache, index);
2723   jvmti_post_field_access(cache, index, is_static, false);
2724 
2725   // Valhalla extras
2726   __ load_unsigned_short(field_index, Address(cache, in_bytes(ResolvedFieldEntry::field_index_offset())));
2727   __ ldr(klass, Address(cache, ResolvedFieldEntry::field_holder_offset()));
2728 
2729   load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
2730 
2731   if (!is_static) {
2732     // obj is on the stack
2733     pop_and_check_object(obj);
2734   }
2735 
2736   // 8179954: We need to make sure that the code generated for
2737   // volatile accesses forms a sequentially-consistent set of
2738   // operations when combined with STLR and LDAR.  Without a leading
2739   // membar it's possible for a simple Dekker test to fail if loads
2740   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
2741   // the stores in one method and we interpret the loads in another.
2742   if (!CompilerConfig::is_c1_or_interpreter_only_no_jvmci()){
2743     Label notVolatile;
2744     __ tbz(flags, ResolvedFieldEntry::is_volatile_shift, notVolatile);
2745     __ membar(MacroAssembler::AnyAny);
2746     __ bind(notVolatile);
2747   }
2748 

2767   __ b(Done);
2768 
2769   __ bind(notByte);
2770   __ cmp(tos_state, (u1)ztos);
2771   __ br(Assembler::NE, notBool);
2772 
2773   // ztos (same code as btos)
2774   __ access_load_at(T_BOOLEAN, IN_HEAP, r0, field, noreg, noreg);
2775   __ push(ztos);
2776   // Rewrite bytecode to be faster
2777   if (rc == may_rewrite) {
2778     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2779     patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2780   }
2781   __ b(Done);
2782 
2783   __ bind(notBool);
2784   __ cmp(tos_state, (u1)atos);
2785   __ br(Assembler::NE, notObj);
2786   // atos
2787   if (!EnableValhalla) {
2788     do_oop_load(_masm, field, r0, IN_HEAP);
2789     __ push(atos);
2790     if (rc == may_rewrite) {
2791       patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2792     }
2793     __ b(Done);
2794   } else { // Valhalla
2795     if (is_static) {
2796       __ load_heap_oop(r0, field, rscratch1, rscratch2);
2797       Label is_null_free_inline_type, uninitialized;
2798       // Issue below if the static field has not been initialized yet
2799       __ test_field_is_null_free_inline_type(flags, noreg /*temp*/, is_null_free_inline_type);
2800         // field is not a null free inline type
2801         __ push(atos);
2802         __ b(Done);
2803       // field is a null free inline type, must not return null even if uninitialized
2804       __ bind(is_null_free_inline_type);
2805         __ cbz(r0, uninitialized);
2806           __ push(atos);
2807           __ b(Done);
2808         __ bind(uninitialized);
2809           Label slow_case, finish;
2810           __ ldrb(rscratch1, Address(klass, InstanceKlass::init_state_offset()));
2811           __ cmp(rscratch1, (u1)InstanceKlass::fully_initialized);
2812           __ br(Assembler::NE, slow_case);
2813           __ get_default_value_oop(klass, off /* temp */, r0);
2814         __ b(finish);
2815         __ bind(slow_case);
2816           __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_static_inline_type_field), obj, cache);
2817           __ bind(finish);
2818           __ verify_oop(r0);
2819           __ push(atos);
2820           __ b(Done);
2821     } else {
2822       Label is_flat, nonnull, is_inline_type, has_null_marker, rewrite_inline;
2823       __ test_field_is_null_free_inline_type(flags, noreg /*temp*/, is_inline_type);
2824       __ test_field_has_null_marker(flags, noreg /*temp*/, has_null_marker);
2825         // Non-inline field case
2826         __ load_heap_oop(r0, field, rscratch1, rscratch2);
2827         __ push(atos);
2828         if (rc == may_rewrite) {
2829           patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2830         }
2831         __ b(Done);
2832       __ bind(is_inline_type);
2833         __ test_field_is_flat(flags, noreg /* temp */, is_flat);
2834          // field is not flat
2835           __ load_heap_oop(r0, field, rscratch1, rscratch2);
2836           __ cbnz(r0, nonnull);
2837             __ get_inline_type_field_klass(klass, field_index, inline_klass);
2838             __ get_default_value_oop(inline_klass, klass /* temp */, r0);
2839           __ bind(nonnull);
2840           __ verify_oop(r0);
2841           __ push(atos);
2842           __ b(rewrite_inline);
2843         __ bind(is_flat);
2844         // field is flat
2845           __ mov(r0, obj);
2846           __ read_flat_field(klass, field_index, off, inline_klass /* temp */, r0);
2847           __ verify_oop(r0);
2848           __ push(atos);
2849           __ b(rewrite_inline);
2850         __ bind(has_null_marker);
2851           call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_nullable_flat_field), obj, cache);
2852           __ verify_oop(r0);
2853           __ push(atos);
2854       __ bind(rewrite_inline);
2855       if (rc == may_rewrite) {
2856         patch_bytecode(Bytecodes::_fast_vgetfield, bc, r1);
2857       }
2858       __ b(Done);
2859     }
2860   }

2861 
2862   __ bind(notObj);
2863   __ cmp(tos_state, (u1)itos);
2864   __ br(Assembler::NE, notInt);
2865   // itos
2866   __ access_load_at(T_INT, IN_HEAP, r0, field, noreg, noreg);
2867   __ push(itos);
2868   // Rewrite bytecode to be faster
2869   if (rc == may_rewrite) {
2870     patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2871   }
2872   __ b(Done);
2873 
2874   __ bind(notInt);
2875   __ cmp(tos_state, (u1)ctos);
2876   __ br(Assembler::NE, notChar);
2877   // ctos
2878   __ access_load_at(T_CHAR, IN_HEAP, r0, field, noreg, noreg);
2879   __ push(ctos);
2880   // Rewrite bytecode to be faster

3001     // c_rarg1: object pointer set up above (null if static)
3002     // c_rarg2: cache entry pointer
3003     // c_rarg3: jvalue object on the stack
3004     __ call_VM(noreg,
3005                CAST_FROM_FN_PTR(address,
3006                                 InterpreterRuntime::post_field_modification),
3007                c_rarg1, c_rarg2, c_rarg3);
3008     __ load_field_entry(cache, index);
3009     __ bind(L1);
3010   }
3011 }
3012 
3013 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
3014   transition(vtos, vtos);
3015 
3016   const Register cache     = r2;
3017   const Register index     = r3;
3018   const Register tos_state = r3;
3019   const Register obj       = r2;
3020   const Register off       = r19;
3021   const Register flags     = r6;
3022   const Register bc        = r4;
3023   const Register inline_klass = r5;
3024 
3025   resolve_cache_and_index_for_field(byte_no, cache, index);
3026   jvmti_post_field_mod(cache, index, is_static);
3027   load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
3028 
3029   Label Done;


3030   {
3031     Label notVolatile;
3032     __ tbz(flags, ResolvedFieldEntry::is_volatile_shift, notVolatile);
3033     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
3034     __ bind(notVolatile);
3035   }
3036 
3037   // field address
3038   const Address field(obj, off);
3039 
3040   Label notByte, notBool, notInt, notShort, notChar,
3041         notLong, notFloat, notObj, notDouble;
3042 
3043   assert(btos == 0, "change code, btos != 0");
3044   __ cbnz(tos_state, notByte);
3045 
3046   // Don't rewrite putstatic, only putfield
3047   if (is_static) rc = may_not_rewrite;
3048 
3049   // btos
3050   {
3051     __ pop(btos);
3052     if (!is_static) pop_and_check_object(obj);

3061   __ cmp(tos_state, (u1)ztos);
3062   __ br(Assembler::NE, notBool);
3063 
3064   // ztos
3065   {
3066     __ pop(ztos);
3067     if (!is_static) pop_and_check_object(obj);
3068     __ access_store_at(T_BOOLEAN, IN_HEAP, field, r0, noreg, noreg, noreg);
3069     if (rc == may_rewrite) {
3070       patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
3071     }
3072     __ b(Done);
3073   }
3074 
3075   __ bind(notBool);
3076   __ cmp(tos_state, (u1)atos);
3077   __ br(Assembler::NE, notObj);
3078 
3079   // atos
3080   {
3081      if (!EnableValhalla) {
3082       __ pop(atos);
3083       if (!is_static) pop_and_check_object(obj);
3084       // Store into the field
3085       do_oop_store(_masm, field, r0, IN_HEAP);
3086       if (rc == may_rewrite) {
3087         patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
3088       }
3089       __ b(Done);
3090      } else { // Valhalla
3091       __ pop(atos);
3092       if (is_static) {
3093         Label is_inline_type;
3094          __ test_field_is_not_null_free_inline_type(flags, noreg /* temp */, is_inline_type);
3095          __ null_check(r0);
3096          __ bind(is_inline_type);
3097          do_oop_store(_masm, field, r0, IN_HEAP);
3098          __ b(Done);
3099       } else {
3100         Label is_inline_type, is_flat, has_null_marker, rewrite_not_inline, rewrite_inline;
3101         __ test_field_is_null_free_inline_type(flags, noreg /*temp*/, is_inline_type);
3102         __ test_field_has_null_marker(flags, noreg /*temp*/, has_null_marker);
3103         // Not an inline type
3104         pop_and_check_object(obj);
3105         // Store into the field
3106         do_oop_store(_masm, field, r0, IN_HEAP);
3107         __ bind(rewrite_not_inline);
3108         if (rc == may_rewrite) {
3109           patch_bytecode(Bytecodes::_fast_aputfield, bc, r19, true, byte_no);
3110         }
3111         __ b(Done);
3112         // Implementation of the inline type semantic
3113         __ bind(is_inline_type);
3114         __ null_check(r0);
3115         __ test_field_is_flat(flags, noreg /*temp*/, is_flat);
3116         // field is not flat
3117         pop_and_check_object(obj);
3118         // Store into the field
3119         do_oop_store(_masm, field, r0, IN_HEAP);
3120         __ b(rewrite_inline);
3121         __ bind(is_flat);
3122         // field is flat
3123         pop_and_check_object(obj);
3124         assert_different_registers(r0, inline_klass, obj, off);
3125         __ load_klass(inline_klass, r0);
3126         __ data_for_oop(r0, r0, inline_klass);
3127         __ add(obj, obj, off);
3128         __ access_value_copy(IN_HEAP, r0, obj, inline_klass);
3129         __ b(rewrite_inline);
3130         __ bind(has_null_marker);
3131         assert_different_registers(r0, cache, r19);
3132         pop_and_check_object(r19);
3133         __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_nullable_flat_field), r19, r0, cache);
3134         __ bind(rewrite_inline);
3135         if (rc == may_rewrite) {
3136           patch_bytecode(Bytecodes::_fast_vputfield, bc, r19, true, byte_no);
3137         }
3138         __ b(Done);
3139       }
3140      }  // Valhalla
3141   }
3142 
3143   __ bind(notObj);
3144   __ cmp(tos_state, (u1)itos);
3145   __ br(Assembler::NE, notInt);
3146 
3147   // itos
3148   {
3149     __ pop(itos);
3150     if (!is_static) pop_and_check_object(obj);
3151     __ access_store_at(T_INT, IN_HEAP, field, r0, noreg, noreg, noreg);
3152     if (rc == may_rewrite) {
3153       patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
3154     }
3155     __ b(Done);
3156   }
3157 
3158   __ bind(notInt);
3159   __ cmp(tos_state, (u1)ctos);
3160   __ br(Assembler::NE, notChar);

3225   {
3226     __ pop(dtos);
3227     if (!is_static) pop_and_check_object(obj);
3228     __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg, noreg);
3229     if (rc == may_rewrite) {
3230       patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
3231     }
3232   }
3233 
3234 #ifdef ASSERT
3235   __ b(Done);
3236 
3237   __ bind(notDouble);
3238   __ stop("Bad state");
3239 #endif
3240 
3241   __ bind(Done);
3242 
3243   {
3244     Label notVolatile;
3245     __ tbz(flags, ResolvedFieldEntry::is_volatile_shift, notVolatile);
3246     __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore);
3247     __ bind(notVolatile);
3248   }
3249 }
3250 
3251 void TemplateTable::putfield(int byte_no)
3252 {
3253   putfield_or_static(byte_no, false);
3254 }
3255 
3256 void TemplateTable::nofast_putfield(int byte_no) {
3257   putfield_or_static(byte_no, false, may_not_rewrite);
3258 }
3259 
3260 void TemplateTable::putstatic(int byte_no) {
3261   putfield_or_static(byte_no, true);
3262 }
3263 
3264 void TemplateTable::jvmti_post_fast_field_mod() {
3265   if (JvmtiExport::can_post_field_modification()) {
3266     // Check to see if a field modification watch has been set before
3267     // we take the time to call into the VM.
3268     Label L2;
3269     __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
3270     __ ldrw(c_rarg3, Address(rscratch1));
3271     __ cbzw(c_rarg3, L2);
3272     __ pop_ptr(r19);                  // copy the object pointer from tos
3273     __ verify_oop(r19);
3274     __ push_ptr(r19);                 // put the object pointer back on tos
3275     // Save tos values before call_VM() clobbers them. Since we have
3276     // to do it for every data type, we use the saved values as the
3277     // jvalue object.
3278     switch (bytecode()) {          // load values into the jvalue object
3279     case Bytecodes::_fast_vputfield: //fall through
3280     case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
3281     case Bytecodes::_fast_bputfield: // fall through
3282     case Bytecodes::_fast_zputfield: // fall through
3283     case Bytecodes::_fast_sputfield: // fall through
3284     case Bytecodes::_fast_cputfield: // fall through
3285     case Bytecodes::_fast_iputfield: __ push_i(r0); break;
3286     case Bytecodes::_fast_dputfield: __ push_d(); break;
3287     case Bytecodes::_fast_fputfield: __ push_f(); break;
3288     case Bytecodes::_fast_lputfield: __ push_l(r0); break;
3289 
3290     default:
3291       ShouldNotReachHere();
3292     }
3293     __ mov(c_rarg3, esp);             // points to jvalue on the stack
3294     // access constant pool cache entry
3295     __ load_field_entry(c_rarg2, r0);
3296     __ verify_oop(r19);
3297     // r19: object pointer copied above
3298     // c_rarg2: cache entry pointer
3299     // c_rarg3: jvalue object on the stack
3300     __ call_VM(noreg,
3301                CAST_FROM_FN_PTR(address,
3302                                 InterpreterRuntime::post_field_modification),
3303                r19, c_rarg2, c_rarg3);
3304 
3305     switch (bytecode()) {             // restore tos values
3306     case Bytecodes::_fast_vputfield: //fall through
3307     case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
3308     case Bytecodes::_fast_bputfield: // fall through
3309     case Bytecodes::_fast_zputfield: // fall through
3310     case Bytecodes::_fast_sputfield: // fall through
3311     case Bytecodes::_fast_cputfield: // fall through
3312     case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
3313     case Bytecodes::_fast_dputfield: __ pop_d(); break;
3314     case Bytecodes::_fast_fputfield: __ pop_f(); break;
3315     case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
3316     default: break;
3317     }
3318     __ bind(L2);
3319   }
3320 }
3321 
3322 void TemplateTable::fast_storefield(TosState state)
3323 {
3324   transition(state, vtos);
3325 
3326   ByteSize base = ConstantPoolCache::base_offset();

3333   // R1: field offset, R2: field holder, R3: flags
3334   load_resolved_field_entry(r2, r2, noreg, r1, r3);
3335 
3336   {
3337     Label notVolatile;
3338     __ tbz(r3, ResolvedFieldEntry::is_volatile_shift, notVolatile);
3339     __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
3340     __ bind(notVolatile);
3341   }
3342 
3343   Label notVolatile;
3344 
3345   // Get object from stack
3346   pop_and_check_object(r2);
3347 
3348   // field address
3349   const Address field(r2, r1);
3350 
3351   // access field
3352   switch (bytecode()) {
3353   case Bytecodes::_fast_vputfield:
3354    {
3355       Label is_flat, has_null_marker, done;
3356       __ test_field_has_null_marker(r3, noreg /* temp */, has_null_marker);
3357       __ null_check(r0);
3358       __ test_field_is_flat(r3, noreg /* temp */, is_flat);
3359       // field is not flat
3360       do_oop_store(_masm, field, r0, IN_HEAP);
3361       __ b(done);
3362       __ bind(is_flat);
3363       // field is flat
3364       __ load_klass(r4, r0);
3365       __ data_for_oop(r0, r0, r4);
3366       __ lea(rscratch1, field);
3367       __ access_value_copy(IN_HEAP, r0, rscratch1, r4);
3368       __ b(done);
3369       __ bind(has_null_marker);
3370       __ load_field_entry(r4, r1);
3371       __ mov(r1, r2);
3372       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_nullable_flat_field), r1, r0, r4);
3373       __ bind(done);
3374     }
3375     break;
3376   case Bytecodes::_fast_aputfield:
3377     do_oop_store(_masm, field, r0, IN_HEAP);
3378     break;
3379   case Bytecodes::_fast_lputfield:
3380     __ access_store_at(T_LONG, IN_HEAP, field, r0, noreg, noreg, noreg);
3381     break;
3382   case Bytecodes::_fast_iputfield:
3383     __ access_store_at(T_INT, IN_HEAP, field, r0, noreg, noreg, noreg);
3384     break;
3385   case Bytecodes::_fast_zputfield:
3386     __ access_store_at(T_BOOLEAN, IN_HEAP, field, r0, noreg, noreg, noreg);
3387     break;
3388   case Bytecodes::_fast_bputfield:
3389     __ access_store_at(T_BYTE, IN_HEAP, field, r0, noreg, noreg, noreg);
3390     break;
3391   case Bytecodes::_fast_sputfield:
3392     __ access_store_at(T_SHORT, IN_HEAP, field, r0, noreg, noreg, noreg);
3393     break;
3394   case Bytecodes::_fast_cputfield:
3395     __ access_store_at(T_CHAR, IN_HEAP, field, r0, noreg, noreg, noreg);

3448   // r0: object
3449   __ verify_oop(r0);
3450   __ null_check(r0);
3451   const Address field(r0, r1);
3452 
3453   // 8179954: We need to make sure that the code generated for
3454   // volatile accesses forms a sequentially-consistent set of
3455   // operations when combined with STLR and LDAR.  Without a leading
3456   // membar it's possible for a simple Dekker test to fail if loads
3457   // use LDR;DMB but stores use STLR.  This can happen if C2 compiles
3458   // the stores in one method and we interpret the loads in another.
3459   if (!CompilerConfig::is_c1_or_interpreter_only_no_jvmci()) {
3460     Label notVolatile;
3461     __ tbz(r3, ResolvedFieldEntry::is_volatile_shift, notVolatile);
3462     __ membar(MacroAssembler::AnyAny);
3463     __ bind(notVolatile);
3464   }
3465 
3466   // access field
3467   switch (bytecode()) {
3468   case Bytecodes::_fast_vgetfield:
3469     {
3470       Register index = r4, klass = r5, inline_klass = r6, tmp = r7;
3471       Label is_flat, has_null_marker, nonnull, Done;
3472       __ test_field_has_null_marker(r3, noreg /*temp*/, has_null_marker);
3473       __ test_field_is_flat(r3, noreg /* temp */, is_flat);
3474         // field is not flat
3475         __ load_heap_oop(r0, field, rscratch1, rscratch2);
3476         __ cbnz(r0, nonnull);
3477           __ load_unsigned_short(index, Address(r2, in_bytes(ResolvedFieldEntry::field_index_offset())));
3478           __ ldr(klass, Address(r2, in_bytes(ResolvedFieldEntry::field_holder_offset())));
3479           __ get_inline_type_field_klass(klass, index, inline_klass);
3480           __ get_default_value_oop(inline_klass, tmp /* temp */, r0);
3481         __ bind(nonnull);
3482         __ verify_oop(r0);
3483         __ b(Done);
3484       __ bind(is_flat);
3485       // field is flat
3486         __ load_unsigned_short(index, Address(r2, in_bytes(ResolvedFieldEntry::field_index_offset())));
3487         __ ldr(klass, Address(r2, in_bytes(ResolvedFieldEntry::field_holder_offset())));
3488         __ read_flat_field(klass, index, r1, tmp /* temp */, r0);
3489         __ verify_oop(r0);
3490         __ b(Done);
3491       __ bind(has_null_marker);
3492         call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_nullable_flat_field), r0, r2);
3493         __ verify_oop(r0);
3494       __ bind(Done);
3495     }
3496     break;
3497   case Bytecodes::_fast_agetfield:
3498     do_oop_load(_masm, field, r0, IN_HEAP);
3499     __ verify_oop(r0);
3500     break;
3501   case Bytecodes::_fast_lgetfield:
3502     __ access_load_at(T_LONG, IN_HEAP, r0, field, noreg, noreg);
3503     break;
3504   case Bytecodes::_fast_igetfield:
3505     __ access_load_at(T_INT, IN_HEAP, r0, field, noreg, noreg);
3506     break;
3507   case Bytecodes::_fast_bgetfield:
3508     __ access_load_at(T_BYTE, IN_HEAP, r0, field, noreg, noreg);
3509     break;
3510   case Bytecodes::_fast_sgetfield:
3511     __ access_load_at(T_SHORT, IN_HEAP, r0, field, noreg, noreg);
3512     break;
3513   case Bytecodes::_fast_cgetfield:
3514     __ access_load_at(T_CHAR, IN_HEAP, r0, field, noreg, noreg);
3515     break;
3516   case Bytecodes::_fast_fgetfield:

3895   Label initialize_header;
3896 
3897   __ get_cpool_and_tags(r4, r0);
3898   // Make sure the class we're about to instantiate has been resolved.
3899   // This is done before loading InstanceKlass to be consistent with the order
3900   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3901   const int tags_offset = Array<u1>::base_offset_in_bytes();
3902   __ lea(rscratch1, Address(r0, r3, Address::lsl(0)));
3903   __ lea(rscratch1, Address(rscratch1, tags_offset));
3904   __ ldarb(rscratch1, rscratch1);
3905   __ cmp(rscratch1, (u1)JVM_CONSTANT_Class);
3906   __ br(Assembler::NE, slow_case);
3907 
3908   // get InstanceKlass
3909   __ load_resolved_klass_at_offset(r4, r3, r4, rscratch1);
3910 
3911   // make sure klass is initialized
3912   assert(VM_Version::supports_fast_class_init_checks(), "Optimization requires support for fast class initialization checks");
3913   __ clinit_barrier(r4, rscratch1, nullptr /*L_fast_path*/, &slow_case);
3914 
3915   __ allocate_instance(r4, r0, r3, r1, true, slow_case);
3916   __ b(done);























































3917 
3918   // slow case
3919   __ bind(slow_case);
3920   __ get_constant_pool(c_rarg1);
3921   __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3922   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3923   __ verify_oop(r0);
3924 
3925   // continue
3926   __ bind(done);
3927   // Must prevent reordering of stores for object initialization with stores that publish the new object.
3928   __ membar(Assembler::StoreStore);
3929 }
3930 
3931 void TemplateTable::newarray() {
3932   transition(itos, atos);
3933   __ load_unsigned_byte(c_rarg1, at_bcp(1));
3934   __ mov(c_rarg2, r0);
3935   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3936           c_rarg1, c_rarg2);

3981   __ bind(quicked);
3982   __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
3983   __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1); // r0 = klass
3984 
3985   __ bind(resolved);
3986   __ load_klass(r19, r3);
3987 
3988   // Generate subtype check.  Blows r2, r5.  Object in r3.
3989   // Superklass in r0.  Subklass in r19.
3990   __ gen_subtype_check(r19, ok_is_subtype);
3991 
3992   // Come here on failure
3993   __ push(r3);
3994   // object is at TOS
3995   __ b(Interpreter::_throw_ClassCastException_entry);
3996 
3997   // Come here on success
3998   __ bind(ok_is_subtype);
3999   __ mov(r0, r3); // Restore object in r3
4000 
4001   __ b(done);
4002   __ bind(is_null);
4003 
4004   // Collect counts on whether this test sees nulls a lot or not.
4005   if (ProfileInterpreter) {


4006     __ profile_null_seen(r2);


4007   }
4008 
4009   __ bind(done);
4010 }
4011 
4012 void TemplateTable::instanceof() {
4013   transition(atos, itos);
4014   Label done, is_null, ok_is_subtype, quicked, resolved;
4015   __ cbz(r0, is_null);
4016 
4017   // Get cpool & tags index
4018   __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
4019   __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
4020   // See if bytecode has already been quicked
4021   __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
4022   __ lea(r1, Address(rscratch1, r19));
4023   __ ldarb(r1, r1);
4024   __ cmp(r1, (u1)JVM_CONSTANT_Class);
4025   __ br(Assembler::EQ, quicked);
4026 
4027   __ push(atos); // save receiver for result, and for GC
4028   call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));

4107 //       in the assembly code structure as well
4108 //
4109 // Stack layout:
4110 //
4111 // [expressions  ] <--- esp               = expression stack top
4112 // ..
4113 // [expressions  ]
4114 // [monitor entry] <--- monitor block top = expression stack bot
4115 // ..
4116 // [monitor entry]
4117 // [frame data   ] <--- monitor block bot
4118 // ...
4119 // [saved rfp    ] <--- rfp
4120 void TemplateTable::monitorenter()
4121 {
4122   transition(atos, vtos);
4123 
4124   // check for null object
4125   __ null_check(r0);
4126 
4127   Label is_inline_type;
4128   __ ldr(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
4129   __ test_markword_is_inline_type(rscratch1, is_inline_type);
4130 
4131   const Address monitor_block_top(
4132         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4133   const Address monitor_block_bot(
4134         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
4135   const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
4136 
4137   Label allocated;
4138 
4139   // initialize entry pointer
4140   __ mov(c_rarg1, zr); // points to free slot or null
4141 
4142   // find a free slot in the monitor block (result in c_rarg1)
4143   {
4144     Label entry, loop, exit;
4145     __ ldr(c_rarg3, monitor_block_top); // derelativize pointer
4146     __ lea(c_rarg3, Address(rfp, c_rarg3, Address::lsl(Interpreter::logStackElementSize)));
4147     // c_rarg3 points to current entry, starting with top-most entry
4148 
4149     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
4150 

4212   // c_rarg1: points to monitor entry
4213   __ bind(allocated);
4214 
4215   // Increment bcp to point to the next bytecode, so exception
4216   // handling for async. exceptions work correctly.
4217   // The object has already been popped from the stack, so the
4218   // expression stack looks correct.
4219   __ increment(rbcp);
4220 
4221   // store object
4222   __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset()));
4223   __ lock_object(c_rarg1);
4224 
4225   // check to make sure this monitor doesn't cause stack overflow after locking
4226   __ save_bcp();  // in case of exception
4227   __ generate_stack_overflow_check(0);
4228 
4229   // The bcp has already been incremented. Just need to dispatch to
4230   // next instruction.
4231   __ dispatch_next(vtos);
4232 
4233   __ bind(is_inline_type);
4234   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4235                     InterpreterRuntime::throw_identity_exception), r0);
4236   __ should_not_reach_here();
4237 }
4238 
4239 
4240 void TemplateTable::monitorexit()
4241 {
4242   transition(atos, vtos);
4243 
4244   // check for null object
4245   __ null_check(r0);
4246 
4247   const int is_inline_type_mask = markWord::inline_type_pattern;
4248   Label has_identity;
4249   __ ldr(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
4250   __ mov(rscratch2, is_inline_type_mask);
4251   __ andr(rscratch1, rscratch1, rscratch2);
4252   __ cmp(rscratch1, rscratch2);
4253   __ br(Assembler::NE, has_identity);
4254   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
4255                      InterpreterRuntime::throw_illegal_monitor_state_exception));
4256   __ should_not_reach_here();
4257   __ bind(has_identity);
4258 
4259   const Address monitor_block_top(
4260         rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
4261   const Address monitor_block_bot(
4262         rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
4263   const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
4264 
4265   Label found;
4266 
4267   // find matching slot
4268   {
4269     Label entry, loop;
4270     __ ldr(c_rarg1, monitor_block_top); // derelativize pointer
4271     __ lea(c_rarg1, Address(rfp, c_rarg1, Address::lsl(Interpreter::logStackElementSize)));
4272     // c_rarg1 points to current entry, starting with top-most entry
4273 
4274     __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
4275                                         // of monitor block
4276     __ b(entry);
4277 
4278     __ bind(loop);
< prev index next >