< prev index next >

src/hotspot/cpu/aarch64/aarch64.ad

Print this page

 3753     Label cont;
 3754     Label object_has_monitor;
 3755     Label no_count;
 3756 
 3757     assert_different_registers(oop, box, tmp, disp_hdr);
 3758 
 3759     // Load markWord from object into displaced_header.
 3760     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3761 
 3762     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3763       __ load_klass(tmp, oop);
 3764       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3765       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3766       __ br(Assembler::NE, cont);
 3767     }
 3768 
 3769     // Check for existing monitor
 3770     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3771 
 3772     if (!UseHeavyMonitors) {
 3773       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3774       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3775 
 3776       // Initialize the box. (Must happen before we update the object mark!)
 3777       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3778 
 3779       // Compare object markWord with an unlocked value (tmp) and if
 3780       // equal exchange the stack address of our box with object markWord.
 3781       // On failure disp_hdr contains the possibly locked markWord.
 3782       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3783                  /*release*/ true, /*weak*/ false, disp_hdr);
 3784       __ br(Assembler::EQ, cont);
 3785 
 3786       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3787 
 3788       // If the compare-and-exchange succeeded, then we found an unlocked
 3789       // object, will have now locked it will continue at label cont
 3790 
 3791       // Check if the owner is self by comparing the value in the
 3792       // markWord of object (disp_hdr) with the stack pointer.
 3793       __ mov(rscratch1, sp);
 3794       __ sub(disp_hdr, disp_hdr, rscratch1);
 3795       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3796       // If condition is true we are cont and hence we can store 0 as the
 3797       // displaced header in the box, which indicates that it is a recursive lock.
 3798       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3799       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3800     } else {
 3801       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3802     }

 3803     __ b(cont);
 3804 
 3805     // Handle existing monitor.
 3806     __ bind(object_has_monitor);
 3807 
 3808     // The object's monitor m is unlocked iff m->owner == NULL,
 3809     // otherwise m->owner may contain a thread or a stack address.
 3810     //
 3811     // Try to CAS m->owner from NULL to current thread.
 3812     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3813     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3814                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3815 
 3816     // Store a non-null value into the box to avoid looking like a re-entrant
 3817     // lock. The fast-path monitor unlock code checks for
 3818     // markWord::monitor_value so use markWord::unused_mark which has the
 3819     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3820     __ mov(tmp, (address)markWord::unused_mark().value());
 3821     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3822 
 3823     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3824 
 3825     __ cmp(rscratch1, rthread);
 3826     __ br(Assembler::NE, cont); // Check for recursive locking
 3827 
 3828     // Recursive lock case
 3829     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3830     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3831 
 3832     __ bind(cont);
 3833     // flag == EQ indicates success
 3834     // flag == NE indicates failure
 3835     __ br(Assembler::NE, no_count);
 3836 
 3837     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 3838 
 3839     __ bind(no_count);
 3840   %}
 3841 
 3842   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3843     C2_MacroAssembler _masm(&cbuf);
 3844     Register oop = as_Register($object$$reg);
 3845     Register box = as_Register($box$$reg);
 3846     Register disp_hdr = as_Register($tmp$$reg);
 3847     Register tmp = as_Register($tmp2$$reg);
 3848     Label cont;
 3849     Label object_has_monitor;
 3850     Label no_count;
 3851 
 3852     assert_different_registers(oop, box, tmp, disp_hdr);
 3853 
 3854     if (!UseHeavyMonitors) {
 3855       // Find the lock address and load the displaced header from the stack.
 3856       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3857 
 3858       // If the displaced header is 0, we have a recursive unlock.
 3859       __ cmp(disp_hdr, zr);
 3860       __ br(Assembler::EQ, cont);
 3861     }
 3862 
 3863     // Handle existing monitor.
 3864     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3865     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);

 3866 
 3867     if (!UseHeavyMonitors) {
 3868       // Check if it is still a light weight lock, this is is true if we
 3869       // see the stack address of the basicLock in the markWord of the
 3870       // object.
 3871 
 3872       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3873                  /*release*/ true, /*weak*/ false, tmp);
 3874     } else {
 3875       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.



 3876     }

 3877     __ b(cont);
 3878 
 3879     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3880 
 3881     // Handle existing monitor.
 3882     __ bind(object_has_monitor);
 3883     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3884     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3885     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));












 3886 
 3887     Label notRecursive;
 3888     __ cbz(disp_hdr, notRecursive);
 3889 
 3890     // Recursive lock
 3891     __ sub(disp_hdr, disp_hdr, 1u);
 3892     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3893     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3894     __ b(cont);
 3895 
 3896     __ bind(notRecursive);
 3897     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3898     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3899     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3900     __ cmp(rscratch1, zr); // Sets flags for result
 3901     __ cbnz(rscratch1, cont);
 3902     // need a release store here
 3903     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3904     __ stlr(zr, tmp); // set unowned
 3905 

 7152   ins_encode(aarch64_enc_ldrw(dst, mem));
 7153 
 7154   ins_pipe(iload_reg_mem);
 7155 %}
 7156 
 7157 // Load Klass Pointer
 7158 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7159 %{
 7160   match(Set dst (LoadKlass mem));
 7161   predicate(!needs_acquiring_load(n));
 7162 
 7163   ins_cost(4 * INSN_COST);
 7164   format %{ "ldr  $dst, $mem\t# class" %}
 7165 
 7166   ins_encode(aarch64_enc_ldr(dst, mem));
 7167 
 7168   ins_pipe(iload_reg_mem);
 7169 %}
 7170 
 7171 // Load Narrow Klass Pointer
 7172 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7173 %{
 7174   match(Set dst (LoadNKlass mem));

 7175   predicate(!needs_acquiring_load(n));
 7176 
 7177   ins_cost(4 * INSN_COST);
 7178   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7179 
 7180   ins_encode(aarch64_enc_ldrw(dst, mem));
 7181 
 7182   ins_pipe(iload_reg_mem);


 7183 %}
 7184 
 7185 // Load Float
 7186 instruct loadF(vRegF dst, memory4 mem)
 7187 %{
 7188   match(Set dst (LoadF mem));
 7189   predicate(!needs_acquiring_load(n));
 7190 
 7191   ins_cost(4 * INSN_COST);
 7192   format %{ "ldrs  $dst, $mem\t# float" %}
 7193 
 7194   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7195 
 7196   ins_pipe(pipe_class_memory);
 7197 %}
 7198 
 7199 // Load Double
 7200 instruct loadD(vRegD dst, memory8 mem)
 7201 %{
 7202   match(Set dst (LoadD mem));

16111 
16112   ins_cost(BRANCH_COST);
16113   // short variant.
16114   // ins_short_branch(1);
16115   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16116 
16117   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16118 
16119   ins_pipe(pipe_branch);
16120 %}
16121 
16122 // counted loop end branch far
16123 // counted loop end branch far unsigned
16124 // TODO: fixme
16125 
16126 // ============================================================================
16127 // inlined locking and unlocking
16128 
16129 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16130 %{
16131   match(Set cr (FastLock object box));
16132   effect(TEMP tmp, TEMP tmp2);
16133 
16134   // TODO
16135   // identify correct cost
16136   ins_cost(5 * INSN_COST);
16137   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16138 
16139   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16140 
16141   ins_pipe(pipe_serial);
16142 %}
16143 
16144 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16145 %{
16146   match(Set cr (FastUnlock object box));
16147   effect(TEMP tmp, TEMP tmp2);
16148 
16149   ins_cost(5 * INSN_COST);
16150   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16151 
16152   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16153 
16154   ins_pipe(pipe_serial);
16155 %}
16156 
16157 
16158 // ============================================================================
16159 // Safepoint Instructions
16160 
16161 // TODO
16162 // provide a near and far version of this code
16163 
16164 instruct safePoint(rFlagsReg cr, iRegP poll)
16165 %{
16166   match(SafePoint poll);
16167   effect(KILL cr);

 3753     Label cont;
 3754     Label object_has_monitor;
 3755     Label no_count;
 3756 
 3757     assert_different_registers(oop, box, tmp, disp_hdr);
 3758 
 3759     // Load markWord from object into displaced_header.
 3760     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3761 
 3762     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3763       __ load_klass(tmp, oop);
 3764       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3765       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3766       __ br(Assembler::NE, cont);
 3767     }
 3768 
 3769     // Check for existing monitor
 3770     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3771 
 3772     if (!UseHeavyMonitors) {
 3773       Label slow;
 3774       __ fast_lock(oop, disp_hdr, box, tmp, rscratch1, slow);
 3775 
 3776       // Indicate success at cont.
 3777       __ cmp(oop, oop);
 3778       __ b(cont);
 3779       __ bind(slow);






















 3780     }
 3781     __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3782     __ b(cont);
 3783 
 3784     // Handle existing monitor.
 3785     __ bind(object_has_monitor);
 3786 
 3787     // The object's monitor m is unlocked iff m->owner == NULL,
 3788     // otherwise m->owner may contain a thread or a stack address.
 3789     //
 3790     // Try to CAS m->owner from NULL to current thread.
 3791     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3792     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3793                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3794 







 3795     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3796 
 3797     __ cmp(rscratch1, rthread);
 3798     __ br(Assembler::NE, cont); // Check for recursive locking
 3799 
 3800     // Recursive lock case
 3801     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3802     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3803 
 3804     __ bind(cont);
 3805     // flag == EQ indicates success
 3806     // flag == NE indicates failure
 3807     __ br(Assembler::NE, no_count);
 3808 
 3809     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 3810 
 3811     __ bind(no_count);
 3812   %}
 3813 
 3814   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3815     C2_MacroAssembler _masm(&cbuf);
 3816     Register oop = as_Register($object$$reg);
 3817     Register box = as_Register($box$$reg);
 3818     Register disp_hdr = as_Register($tmp$$reg);
 3819     Register tmp = as_Register($tmp2$$reg);
 3820     Label cont;
 3821     Label object_has_monitor;
 3822     Label no_count;
 3823 
 3824     assert_different_registers(oop, box, tmp, disp_hdr);
 3825 









 3826     // Handle existing monitor.
 3827     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3828 
 3829     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3830 
 3831     if (!UseHeavyMonitors) {
 3832       Label slow;
 3833       __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);

 3834 
 3835       __ fast_unlock(oop, tmp, box, disp_hdr, slow);
 3836 
 3837       // Indicate success at cont.
 3838       __ cmp(oop, oop);
 3839       __ b(cont);
 3840 
 3841       __ bind(slow);
 3842     }
 3843     __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3844     __ b(cont);
 3845 


 3846     // Handle existing monitor.
 3847     __ bind(object_has_monitor);
 3848     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3849     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3850 
 3851     // If the owner is anonymous, we need to fix it -- in the slow-path.
 3852     {
 3853       Label L;
 3854       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3855       __ cmp(disp_hdr, (unsigned char)(intptr_t) ANONYMOUS_OWNER);
 3856       __ br(Assembler::NE, L);
 3857       __ tst(oop, oop); // Indicate failure at cont -- dive into slow-path.
 3858       __ b(cont);
 3859       __ bind(L);
 3860     }
 3861 
 3862    __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3863 
 3864     Label notRecursive;
 3865     __ cbz(disp_hdr, notRecursive);
 3866 
 3867     // Recursive lock
 3868     __ sub(disp_hdr, disp_hdr, 1u);
 3869     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3870     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3871     __ b(cont);
 3872 
 3873     __ bind(notRecursive);
 3874     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3875     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3876     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3877     __ cmp(rscratch1, zr); // Sets flags for result
 3878     __ cbnz(rscratch1, cont);
 3879     // need a release store here
 3880     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3881     __ stlr(zr, tmp); // set unowned
 3882 

 7129   ins_encode(aarch64_enc_ldrw(dst, mem));
 7130 
 7131   ins_pipe(iload_reg_mem);
 7132 %}
 7133 
 7134 // Load Klass Pointer
 7135 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7136 %{
 7137   match(Set dst (LoadKlass mem));
 7138   predicate(!needs_acquiring_load(n));
 7139 
 7140   ins_cost(4 * INSN_COST);
 7141   format %{ "ldr  $dst, $mem\t# class" %}
 7142 
 7143   ins_encode(aarch64_enc_ldr(dst, mem));
 7144 
 7145   ins_pipe(iload_reg_mem);
 7146 %}
 7147 
 7148 // Load Narrow Klass Pointer
 7149 instruct loadNKlass(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
 7150 %{
 7151   match(Set dst (LoadNKlass mem));
 7152   effect(TEMP_DEF dst, KILL cr);
 7153   predicate(!needs_acquiring_load(n));
 7154 
 7155   ins_cost(4 * INSN_COST);
 7156   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7157   ins_encode %{
 7158     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
 7159     assert($mem$$index$$Register == noreg, "expect no index");
 7160     __ load_nklass($dst$$Register, $mem$$base$$Register);
 7161   %}
 7162   ins_pipe(pipe_slow);
 7163 %}
 7164 
 7165 // Load Float
 7166 instruct loadF(vRegF dst, memory4 mem)
 7167 %{
 7168   match(Set dst (LoadF mem));
 7169   predicate(!needs_acquiring_load(n));
 7170 
 7171   ins_cost(4 * INSN_COST);
 7172   format %{ "ldrs  $dst, $mem\t# float" %}
 7173 
 7174   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7175 
 7176   ins_pipe(pipe_class_memory);
 7177 %}
 7178 
 7179 // Load Double
 7180 instruct loadD(vRegD dst, memory8 mem)
 7181 %{
 7182   match(Set dst (LoadD mem));

16091 
16092   ins_cost(BRANCH_COST);
16093   // short variant.
16094   // ins_short_branch(1);
16095   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16096 
16097   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16098 
16099   ins_pipe(pipe_branch);
16100 %}
16101 
16102 // counted loop end branch far
16103 // counted loop end branch far unsigned
16104 // TODO: fixme
16105 
16106 // ============================================================================
16107 // inlined locking and unlocking
16108 
16109 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16110 %{
16111   match(Set cr (FastLock object));
16112   effect(TEMP tmp, TEMP tmp2, TEMP box);
16113 
16114   // TODO
16115   // identify correct cost
16116   ins_cost(5 * INSN_COST);
16117   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16118 
16119   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16120 
16121   ins_pipe(pipe_serial);
16122 %}
16123 
16124 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16125 %{
16126   match(Set cr (FastUnlock object));
16127   effect(TEMP box, TEMP tmp, TEMP tmp2);
16128 
16129   ins_cost(5 * INSN_COST);
16130   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16131 
16132   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16133 
16134   ins_pipe(pipe_serial);
16135 %}
16136 
16137 
16138 // ============================================================================
16139 // Safepoint Instructions
16140 
16141 // TODO
16142 // provide a near and far version of this code
16143 
16144 instruct safePoint(rFlagsReg cr, iRegP poll)
16145 %{
16146   match(SafePoint poll);
16147   effect(KILL cr);
< prev index next >