< prev index next > src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
Print this page
decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
bind(no_count);
}
! void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
Register t2, Register t3) {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
! assert_different_registers(obj, t1, t2, t3);
// Handle inflated monitor.
Label inflated;
// Finish fast lock successfully. MUST branch to with flag == EQ
Label locked;
// Finish fast lock unsuccessfully. MUST branch to with flag == NE
Label slow_path;
if (DiagnoseSyncOnValueBasedClasses != 0) {
load_klass(t1, obj);
ldrw(t1, Address(t1, Klass::access_flags_offset()));
tstw(t1, JVM_ACC_IS_VALUE_BASED_CLASS);
br(Assembler::NE, slow_path);
}
const Register t1_mark = t1;
{ // Lightweight locking
// Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
Label push;
const Register t2_top = t2;
- const Register t3_t = t3;
// Check if lock-stack is full.
ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
cmpw(t2_top, (unsigned)LockStack::end_offset() - 1);
br(Assembler::GT, slow_path);
decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
bind(no_count);
}
! void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register t1,
Register t2, Register t3) {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
! assert_different_registers(obj, box, t1, t2, t3);
// Handle inflated monitor.
Label inflated;
// Finish fast lock successfully. MUST branch to with flag == EQ
Label locked;
// Finish fast lock unsuccessfully. MUST branch to with flag == NE
Label slow_path;
+ // Clear box. TODO[OMWorld]: Is this necessary? May also defer this to not write twice.
+ str(zr, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
+
if (DiagnoseSyncOnValueBasedClasses != 0) {
load_klass(t1, obj);
ldrw(t1, Address(t1, Klass::access_flags_offset()));
tstw(t1, JVM_ACC_IS_VALUE_BASED_CLASS);
br(Assembler::NE, slow_path);
}
const Register t1_mark = t1;
+ const Register t3_t = t3;
{ // Lightweight locking
// Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
Label push;
const Register t2_top = t2;
// Check if lock-stack is full.
ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
cmpw(t2_top, (unsigned)LockStack::end_offset() - 1);
br(Assembler::GT, slow_path);
}
{ // Handle inflated monitor.
bind(inflated);
! // mark contains the tagged ObjectMonitor*.
! const Register t1_tagged_monitor = t1_mark;
! const uintptr_t monitor_tag = markWord::monitor_value;
! const Register t2_owner_addr = t2;
! const Register t3_owner = t3;
! // Compute owner address.
- lea(t2_owner_addr, Address(t1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
! // CAS owner (null => current thread).
! cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
- /*release*/ false, /*weak*/ false, t3_owner);
- br(Assembler::EQ, locked);
! // Check if recursive.
! cmp(t3_owner, rthread);
! br(Assembler::NE, slow_path);
- // Recursive.
- increment(Address(t1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1);
}
bind(locked);
increment(Address(rthread, JavaThread::held_monitor_count_offset()));
}
{ // Handle inflated monitor.
bind(inflated);
! if (!OMUseC2Cache) {
! // Set Flags == NE
! cmp(zr, obj);
! b(slow_path);
! } else {
+
+ if (OMCacheHitRate) increment(Address(rthread, JavaThread::lock_lookup_offset()));
! Label monitor_found;
! // Load cache address
! lea(t3_t, Address(rthread, JavaThread::om_cache_oops_offset()));
! const int num_unrolled = MIN2(OMC2UnrollCacheEntries, OMCacheSize);
! for (int i = 0; i < num_unrolled; i++) {
! ldr(t1, Address(t3_t));
+ cmp(obj, t1);
+ br(Assembler::EQ, monitor_found);
+ if (i + 1 != num_unrolled) {
+ increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
+ }
+ }
+
+ if (num_unrolled == 0 || (OMC2UnrollCacheLookupLoopTail && num_unrolled != OMCacheSize)) {
+ if (num_unrolled != 0) {
+ // Loop after unrolling, advance iterator.
+ increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
+ }
+
+ Label loop;
+
+ // Search for obj in cache.
+ bind(loop);
+
+ // Check for match.
+ ldr(t1, Address(t3_t));
+ cmp(obj, t1);
+ br(Assembler::EQ, monitor_found);
+
+ // Search until null encountered, guaranteed _null_sentinel at end.
+ increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
+ cbnz(t1, loop);
+ // Cache Miss, NE set from cmp above, cbnz does not set flags
+ b(slow_path);
+ } else {
+ b(slow_path);
+ }
+
+ bind(monitor_found);
+ ldr(t1, Address(t3_t, OMCache::oop_to_monitor_difference()));
+ if (OMCacheHitRate) increment(Address(rthread, JavaThread::lock_hit_offset()));
+
+ // ObjectMonitor* is in t1
+ const Register t1_monitor = t1;
+ const Register t2_owner_addr = t2;
+ const Register t3_owner = t3;
+
+ Label recursive;
+ Label monitor_locked;
+
+ // Compute owner address.
+ lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
+
+ if (OMRecursiveFastPath) {
+ ldr(t3_owner, Address(t2_owner_addr));
+ cmp(t3_owner, rthread);
+ br(Assembler::EQ, recursive);
+ }
+
+ // CAS owner (null => current thread).
+ cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
+ /*release*/ false, /*weak*/ false, t3_owner);
+ br(Assembler::EQ, monitor_locked);
+
+ if (OMRecursiveFastPath) {
+ b(slow_path);
+ } else {
+ // Check if recursive.
+ cmp(t3_owner, rthread);
+ br(Assembler::NE, slow_path);
+ }
+
+ // Recursive.
+ bind(recursive);
+ increment(Address(t1_monitor, ObjectMonitor::recursions_offset()), 1);
+
+ bind(monitor_locked);
+ str(t1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
+ }
}
bind(locked);
increment(Address(rthread, JavaThread::held_monitor_count_offset()));
bind(flag_correct);
#endif
// C2 uses the value of Flags (NE vs EQ) to determine the continuation.
}
! void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Register t2,
! Register t3) {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
! assert_different_registers(obj, t1, t2, t3);
// Handle inflated monitor.
Label inflated, inflated_load_monitor;
// Finish fast unlock successfully. MUST branch to with flag == EQ
Label unlocked;
bind(flag_correct);
#endif
// C2 uses the value of Flags (NE vs EQ) to determine the continuation.
}
! void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Register t1,
! Register t2, Register t3) {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
! assert_different_registers(obj, box, t1, t2, t3);
// Handle inflated monitor.
Label inflated, inflated_load_monitor;
// Finish fast unlock successfully. MUST branch to with flag == EQ
Label unlocked;
const Register t2_top = t2;
const Register t3_t = t3;
{ // Lightweight unlock
+ Label push_and_slow_path;
+
// Check if obj is top of lock-stack.
ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
subw(t2_top, t2_top, oopSize);
ldr(t3_t, Address(rthread, t2_top));
cmp(obj, t3_t);
// Not recursive.
// Load Mark.
ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
// Check header for monitor (0b10).
! tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
// Try to unlock. Transition lock bits 0b00 => 0b01
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
orr(t3_t, t1_mark, markWord::unlocked_value);
cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
/*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
br(Assembler::EQ, unlocked);
// Compare and exchange failed.
// Restore lock-stack and handle the unlock in runtime.
DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
addw(t2_top, t2_top, oopSize);
str(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
// Not recursive.
// Load Mark.
ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
// Check header for monitor (0b10).
! // Because we got here by popping (meaning we pushed in locked)
+ // there will be no monitor in the box. So we need to push back the obj
+ // so that the runtime can fix any potential anonymous owner.
+ tbnz(t1_mark, exact_log2(markWord::monitor_value), push_and_slow_path);
// Try to unlock. Transition lock bits 0b00 => 0b01
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
orr(t3_t, t1_mark, markWord::unlocked_value);
cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
/*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
br(Assembler::EQ, unlocked);
+ bind(push_and_slow_path);
// Compare and exchange failed.
// Restore lock-stack and handle the unlock in runtime.
DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
addw(t2_top, t2_top, oopSize);
str(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
br(Assembler::NE, inflated);
stop("Fast Unlock lock on stack");
bind(check_done);
#endif
! // mark contains the tagged ObjectMonitor*.
! const Register t1_monitor = t1_mark;
! const uintptr_t monitor_tag = markWord::monitor_value;
!
! // Untag the monitor.
! sub(t1_monitor, t1_mark, monitor_tag);
!
! const Register t2_recursions = t2;
! Label not_recursive;
!
! // Check if recursive.
! ldr(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
! cbz(t2_recursions, not_recursive);
!
! // Recursive unlock.
! sub(t2_recursions, t2_recursions, 1u);
! str(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
! // Set flag == EQ
! cmp(t2_recursions, t2_recursions);
! b(unlocked);
!
! bind(not_recursive);
!
! Label release;
! const Register t2_owner_addr = t2;
!
! // Compute owner address.
! lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
!
! // Check if the entry lists are empty.
! ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
! ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
! orr(rscratch1, rscratch1, t3_t);
! cmp(rscratch1, zr);
! br(Assembler::EQ, release);
!
! // The owner may be anonymous and we removed the last obj entry in
! // the lock-stack. This loses the information about the owner.
! // Write the thread to the owner field so the runtime knows the owner.
! str(rthread, Address(t2_owner_addr));
! b(slow_path);
!
! bind(release);
! // Set owner to null.
! // Release to satisfy the JMM
! stlr(zr, t2_owner_addr);
}
bind(unlocked);
decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
br(Assembler::NE, inflated);
stop("Fast Unlock lock on stack");
bind(check_done);
#endif
! if (!OMUseC2Cache) {
! b(slow_path);
! } else {
! const Register t1_monitor = t1;
!
! if (OMCacheHitRate) increment(Address(rthread, JavaThread::unlock_lookup_offset()));
! ldr(t1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
! // TODO: Cleanup these constants (with an enum and asserts)
! cmp(t1_monitor, (uint8_t)2);
! // Non symmetrical, take slow path monitor == 0 or 1, 0 and 1 < 2, both LS and NE
! br(Assembler::LO, slow_path);
! if (OMCacheHitRate) increment(Address(rthread, JavaThread::unlock_hit_offset()));
!
! const Register t2_recursions = t2;
! Label not_recursive;
!
! // Check if recursive.
! ldr(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
! cbz(t2_recursions, not_recursive);
!
! // Recursive unlock.
! sub(t2_recursions, t2_recursions, 1u);
! str(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
! // Set flag == EQ
! cmp(t2_recursions, t2_recursions);
! b(unlocked);
!
! bind(not_recursive);
!
! Label release;
! const Register t2_owner_addr = t2;
!
! // Compute owner address.
! lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
!
! // Check if the entry lists are empty.
! ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
! ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
! orr(rscratch1, rscratch1, t3_t);
! cmp(rscratch1, zr);
! br(Assembler::EQ, release);
!
! // The owner may be anonymous and we removed the last obj entry in
! // the lock-stack. This loses the information about the owner.
! // Write the thread to the owner field so the runtime knows the owner.
! str(rthread, Address(t2_owner_addr));
+ b(slow_path);
+
+ bind(release);
+ // Set owner to null.
+ // Release to satisfy the JMM
+ stlr(zr, t2_owner_addr);
+ }
}
bind(unlocked);
decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
return true;
}
}
return MacroAssembler::in_scratch_emit_size();
}
+
+ void C2_MacroAssembler::load_nklass_compact(Register dst, Register obj, Register index, int scale, int disp) {
+ // Note: Don't clobber obj anywhere in that method!
+
+ // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
+ // obj-start, so that we can load from the object's mark-word instead. Usually the address
+ // comes as obj-start in obj and klass_offset_in_bytes in disp. However, sometimes C2
+ // emits code that pre-computes obj-start + klass_offset_in_bytes into a register, and
+ // then passes that register as obj and 0 in disp. The following code extracts the base
+ // and offset to load the mark-word.
+ int offset = oopDesc::mark_offset_in_bytes() + disp - oopDesc::klass_offset_in_bytes();
+ if (index == noreg) {
+ ldr(dst, Address(obj, offset));
+ } else {
+ lea(dst, Address(obj, index, Address::lsl(scale)));
+ ldr(dst, Address(dst, offset));
+ }
+ lsr(dst, dst, markWord::klass_shift);
+ }
< prev index next >