< prev index next > src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
Print this page
#include "runtime/basicLock.hpp"
#include "runtime/os.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
! int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction");
! assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
- Label done;
int null_check_offset = -1;
verify_oop(obj);
// save object being locked into the BasicObjectLock
#include "runtime/basicLock.hpp"
#include "runtime/os.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
! int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction");
! assert_different_registers(hdr, obj, disp_hdr, tmp);
int null_check_offset = -1;
verify_oop(obj);
// save object being locked into the BasicObjectLock
jcc(Assembler::notZero, slow_case);
}
// Load object header
movptr(hdr, Address(obj, hdr_offset));
! // and mark it as unlocked
! orptr(hdr, markWord::unlocked_value);
! // save unlocked object header into the displaced header location on the stack
! movptr(Address(disp_hdr, 0), hdr);
! // test if object header is still the same (i.e. unlocked), and if so, store the
! // displaced header address in the object header - if it is not the same, get the
! // object header instead
! MacroAssembler::lock(); // must be immediately before cmpxchg!
! cmpxchgptr(disp_hdr, Address(obj, hdr_offset));
! // if the object header was the same, we're done
! jcc(Assembler::equal, done);
! // if the object header was not the same, it is now in the hdr register
! // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
! //
! // 1) (hdr & aligned_mask) == 0
! // 2) rsp <= hdr
! // 3) hdr <= rsp + page_size
! //
! // these 3 tests can be done by evaluating the following expression:
! //
! // (hdr - rsp) & (aligned_mask - page_size)
! //
! // assuming both the stack pointer and page_size have their least
! // significant 2 bits cleared and page_size is a power of 2
! subptr(hdr, rsp);
! andptr(hdr, aligned_mask - (int)os::vm_page_size());
! // for recursive locking, the result is zero => save it in the displaced header
! // location (NULL in the displaced hdr location indicates recursive locking)
! movptr(Address(disp_hdr, 0), hdr);
! // otherwise we don't care about the result and handle locking via runtime call
! jcc(Assembler::notZero, slow_case);
! // done
! bind(done);
inc_held_monitor_count();
return null_check_offset;
}
jcc(Assembler::notZero, slow_case);
}
// Load object header
movptr(hdr, Address(obj, hdr_offset));
!
! if (UseFastLocking) {
! #ifdef _LP64
! const Register thread = r15_thread;
! #else
! const Register thread = disp_hdr;
! get_thread(thread);
! #endif
! fast_lock_impl(obj, hdr, thread, tmp, slow_case, LP64_ONLY(false) NOT_LP64(true));
! } else {
! Label done;
! orptr(hdr, markWord::unlocked_value);
! // save unlocked object header into the displaced header location on the stack
! movptr(Address(disp_hdr, 0), hdr);
! // test if object header is still the same (i.e. unlocked), and if so, store the
! // displaced header address in the object header - if it is not the same, get the
! // object header instead
! MacroAssembler::lock(); // must be immediately before cmpxchg!
! cmpxchgptr(disp_hdr, Address(obj, hdr_offset));
! // if the object header was the same, we're done
! jcc(Assembler::equal, done);
! // if the object header was not the same, it is now in the hdr register
! // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
! //
! // 1) (hdr & aligned_mask) == 0
! // 2) rsp <= hdr
! // 3) hdr <= rsp + page_size
! //
! // these 3 tests can be done by evaluating the following expression:
! //
! // (hdr - rsp) & (aligned_mask - page_size)
! //
! // assuming both the stack pointer and page_size have their least
+ // significant 2 bits cleared and page_size is a power of 2
+ subptr(hdr, rsp);
+ andptr(hdr, aligned_mask - (int)os::vm_page_size());
+ // for recursive locking, the result is zero => save it in the displaced header
+ // location (NULL in the displaced hdr location indicates recursive locking)
+ movptr(Address(disp_hdr, 0), hdr);
+ // otherwise we don't care about the result and handle locking via runtime call
+ jcc(Assembler::notZero, slow_case);
+ // done
+ bind(done);
+ }
inc_held_monitor_count();
return null_check_offset;
}
const int hdr_offset = oopDesc::mark_offset_in_bytes();
assert(disp_hdr == rax, "disp_hdr must be rax, for the cmpxchg instruction");
assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
Label done;
! // load displaced header
! movptr(hdr, Address(disp_hdr, 0));
! // if the loaded hdr is NULL we had recursive locking
! testptr(hdr, hdr);
! // if we had recursive locking, we are done
! jcc(Assembler::zero, done);
// load object
movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
-
verify_oop(obj);
- // test if object header is pointing to the displaced header, and if so, restore
- // the displaced header in the object - if the object header is not pointing to
- // the displaced header, get the object header instead
- MacroAssembler::lock(); // must be immediately before cmpxchg!
- cmpxchgptr(hdr, Address(obj, hdr_offset));
- // if the object header was not pointing to the displaced header,
- // we do unlocking via runtime call
- jcc(Assembler::notEqual, slow_case);
- // done
- bind(done);
dec_held_monitor_count();
}
// Defines obj, preserves var_size_in_bytes
const int hdr_offset = oopDesc::mark_offset_in_bytes();
assert(disp_hdr == rax, "disp_hdr must be rax, for the cmpxchg instruction");
assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
Label done;
! if (!UseFastLocking) {
! // load displaced header
! movptr(hdr, Address(disp_hdr, 0));
! // if the loaded hdr is NULL we had recursive locking
! testptr(hdr, hdr);
! // if we had recursive locking, we are done
+ jcc(Assembler::zero, done);
+ }
+
// load object
movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
verify_oop(obj);
+ if (UseFastLocking) {
+ movptr(disp_hdr, Address(obj, hdr_offset));
+ andptr(disp_hdr, ~(int32_t)markWord::lock_mask_in_place);
+ fast_unlock_impl(obj, disp_hdr, hdr, slow_case);
+ } else {
+ // test if object header is pointing to the displaced header, and if so, restore
+ // the displaced header in the object - if the object header is not pointing to
+ // the displaced header, get the object header instead
+ MacroAssembler::lock(); // must be immediately before cmpxchg!
+ cmpxchgptr(hdr, Address(obj, hdr_offset));
+ // if the object header was not pointing to the displaced header,
+ // we do unlocking via runtime call
+ jcc(Assembler::notEqual, slow_case);
+ // done
+ }
+ bind(done);
dec_held_monitor_count();
}
// Defines obj, preserves var_size_in_bytes
}
}
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
! assert_different_registers(obj, klass, len);
! movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
#ifdef _LP64
! if (UseCompressedClassPointers) { // Take care not to kill klass
! movptr(t1, klass);
! encode_klass_not_null(t1, rscratch1);
! movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
! } else
#endif
! {
! movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
}
-
if (len->is_valid()) {
movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len);
}
#ifdef _LP64
! else if (UseCompressedClassPointers) {
xorptr(t1, t1);
store_klass_gap(obj, t1);
}
#endif
}
}
}
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
! assert_different_registers(obj, klass, len, t1, t2);
! if (UseCompactObjectHeaders) {
+ movptr(t1, Address(klass, Klass::prototype_header_offset()));
+ movptr(Address(obj, oopDesc::mark_offset_in_bytes()), t1);
+ } else {
+ movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
#ifdef _LP64
! if (UseCompressedClassPointers) { // Take care not to kill klass
! movptr(t1, klass);
! encode_klass_not_null(t1, rscratch1);
! movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
! } else
#endif
! {
! movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
+ }
}
if (len->is_valid()) {
movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len);
}
#ifdef _LP64
! else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
xorptr(t1, t1);
store_klass_gap(obj, t1);
}
#endif
}
if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {
// clear rest of allocated space
const Register t1_zero = t1;
const Register index = t2;
const int threshold = 6 * BytesPerWord; // approximate break even point for code size (see comments below)
if (var_size_in_bytes != noreg) {
mov(index, var_size_in_bytes);
! initialize_body(obj, index, hdr_size_in_bytes, t1_zero);
} else if (con_size_in_bytes <= threshold) {
// use explicit null stores
// code size = 2 + 3*n bytes (n = number of fields to clear)
xorptr(t1_zero, t1_zero); // use t1_zero reg to clear memory (shorter code)
! for (int i = hdr_size_in_bytes; i < con_size_in_bytes; i += BytesPerWord)
movptr(Address(obj, i), t1_zero);
! } else if (con_size_in_bytes > hdr_size_in_bytes) {
// use loop to null out the fields
// code size = 16 bytes for even n (n = number of fields to clear)
// initialize last object field first if odd number of fields
xorptr(t1_zero, t1_zero); // use t1_zero reg to clear memory (shorter code)
! movptr(index, (con_size_in_bytes - hdr_size_in_bytes) >> 3);
// initialize last object field if constant size is odd
! if (((con_size_in_bytes - hdr_size_in_bytes) & 4) != 0)
movptr(Address(obj, con_size_in_bytes - (1*BytesPerWord)), t1_zero);
// initialize remaining object fields: rdx is a multiple of 2
{ Label loop;
bind(loop);
! movptr(Address(obj, index, Address::times_8, hdr_size_in_bytes - (1*BytesPerWord)),
t1_zero);
! NOT_LP64(movptr(Address(obj, index, Address::times_8, hdr_size_in_bytes - (2*BytesPerWord)),
t1_zero);)
decrement(index);
jcc(Assembler::notZero, loop);
}
}
if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {
// clear rest of allocated space
const Register t1_zero = t1;
const Register index = t2;
const int threshold = 6 * BytesPerWord; // approximate break even point for code size (see comments below)
+ int hdr_size_aligned = align_up(hdr_size_in_bytes, BytesPerWord); // klass gap is already cleared by init_header().
if (var_size_in_bytes != noreg) {
mov(index, var_size_in_bytes);
! initialize_body(obj, index, hdr_size_aligned, t1_zero);
} else if (con_size_in_bytes <= threshold) {
// use explicit null stores
// code size = 2 + 3*n bytes (n = number of fields to clear)
xorptr(t1_zero, t1_zero); // use t1_zero reg to clear memory (shorter code)
! for (int i = hdr_size_aligned; i < con_size_in_bytes; i += BytesPerWord)
movptr(Address(obj, i), t1_zero);
! } else if (con_size_in_bytes > hdr_size_aligned) {
// use loop to null out the fields
// code size = 16 bytes for even n (n = number of fields to clear)
// initialize last object field first if odd number of fields
xorptr(t1_zero, t1_zero); // use t1_zero reg to clear memory (shorter code)
! movptr(index, (con_size_in_bytes - hdr_size_aligned) >> 3);
// initialize last object field if constant size is odd
! if (((con_size_in_bytes - hdr_size_aligned) & 4) != 0)
movptr(Address(obj, con_size_in_bytes - (1*BytesPerWord)), t1_zero);
// initialize remaining object fields: rdx is a multiple of 2
{ Label loop;
bind(loop);
! movptr(Address(obj, index, Address::times_8, hdr_size_aligned - (1*BytesPerWord)),
t1_zero);
! NOT_LP64(movptr(Address(obj, index, Address::times_8, hdr_size_aligned - (2*BytesPerWord)),
t1_zero);)
decrement(index);
jcc(Assembler::notZero, loop);
}
}
}
verify_oop(obj);
}
! void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, Address::ScaleFactor f, Register klass, Label& slow_case) {
assert(obj == rax, "obj must be in rax, for cmpxchg");
assert_different_registers(obj, len, t1, t2, klass);
// determine alignment mask
assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
}
verify_oop(obj);
}
! void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case) {
assert(obj == rax, "obj must be in rax, for cmpxchg");
assert_different_registers(obj, len, t1, t2, klass);
// determine alignment mask
assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
cmpptr(len, checked_cast<int32_t>(max_array_allocation_length));
jcc(Assembler::above, slow_case);
const Register arr_size = t2; // okay to be the same
// align object end
! movptr(arr_size, header_size * BytesPerWord + MinObjAlignmentInBytesMask);
lea(arr_size, Address(arr_size, len, f));
andptr(arr_size, ~MinObjAlignmentInBytesMask);
try_allocate(obj, arr_size, 0, t1, t2, slow_case);
initialize_header(obj, klass, len, t1, t2);
// clear rest of allocated space
const Register len_zero = len;
! initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
if (CURRENT_ENV->dtrace_alloc_probes()) {
assert(obj == rax, "must be");
call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
}
cmpptr(len, checked_cast<int32_t>(max_array_allocation_length));
jcc(Assembler::above, slow_case);
const Register arr_size = t2; // okay to be the same
// align object end
! movptr(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
lea(arr_size, Address(arr_size, len, f));
andptr(arr_size, ~MinObjAlignmentInBytesMask);
try_allocate(obj, arr_size, 0, t1, t2, slow_case);
initialize_header(obj, klass, len, t1, t2);
// clear rest of allocated space
const Register len_zero = len;
! initialize_body(obj, arr_size, base_offset_in_bytes, len_zero);
if (CURRENT_ENV->dtrace_alloc_probes()) {
assert(obj == rax, "must be");
call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
}
const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
assert(UseCompressedClassPointers || offset() - start_offset == ic_cmp_size, "check alignment in emit_method_entry");
}
! void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) {
assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
// Make sure there is enough stack space for this method's activation.
// Note that we do this before doing an enter(). This matches the
// ordering of C2's stack overflow check / rsp decrement and allows
// the SharedRuntime stack overflow handling to be consistent
const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
assert(UseCompressedClassPointers || offset() - start_offset == ic_cmp_size, "check alignment in emit_method_entry");
}
! void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes, int max_monitors) {
assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
// Make sure there is enough stack space for this method's activation.
// Note that we do this before doing an enter(). This matches the
// ordering of C2's stack overflow check / rsp decrement and allows
// the SharedRuntime stack overflow handling to be consistent
empty_FPU_stack();
}
#endif // !_LP64 && COMPILER2
decrement(rsp, frame_size_in_bytes); // does not emit code for frame_size == 0
+ #ifdef _LP64
+ if (UseFastLocking && max_monitors > 0) {
+ Label ok;
+ movptr(rax, Address(r15_thread, JavaThread::lock_stack_current_offset()));
+ addptr(rax, max_monitors * wordSize);
+ cmpptr(rax, Address(r15_thread, JavaThread::lock_stack_limit_offset()));
+ jcc(Assembler::less, ok);
+ assert(StubRoutines::x86::check_lock_stack() != NULL, "need runtime call stub");
+ call(RuntimeAddress(StubRoutines::x86::check_lock_stack()));
+ bind(ok);
+ }
+ #endif
+
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
// C1 code is not hot enough to micro optimize the nmethod entry barrier with an out-of-line stub
bs->nmethod_entry_barrier(this, NULL /* slow_path */, NULL /* continuation */);
}
< prev index next >