< prev index next > src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
Print this page
/*
! * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
/*
! * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
cset(result, NE); // Not equal or unordered
cneg(result, result, LO); // Less than
}
}
! int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register scratch, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
! assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
Label done;
int null_check_offset = -1;
verify_oop(obj);
cset(result, NE); // Not equal or unordered
cneg(result, result, LO); // Less than
}
}
! int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
! assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
Label done;
int null_check_offset = -1;
verify_oop(obj);
tstw(hdr, JVM_ACC_IS_VALUE_BASED_CLASS);
br(Assembler::NE, slow_case);
}
if (UseBiasedLocking) {
! assert(scratch != noreg, "should have scratch register at this point");
! biased_locking_enter(disp_hdr, obj, hdr, scratch, false, done, &slow_case);
}
! // Load object header
! ldr(hdr, Address(obj, hdr_offset));
! // and mark it as unlocked
! orr(hdr, hdr, markWord::unlocked_value);
! // save unlocked object header into the displaced header location on the stack
! str(hdr, Address(disp_hdr, 0));
! // test if object header is still the same (i.e. unlocked), and if so, store the
! // displaced header address in the object header - if it is not the same, get the
! // object header instead
! lea(rscratch2, Address(obj, hdr_offset));
! cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
! // if the object header was the same, we're done
! // if the object header was not the same, it is now in the hdr register
! // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
! //
! // 1) (hdr & aligned_mask) == 0
! // 2) sp <= hdr
! // 3) hdr <= sp + page_size
! //
! // these 3 tests can be done by evaluating the following expression:
! //
! // (hdr - sp) & (aligned_mask - page_size)
! //
! // assuming both the stack pointer and page_size have their least
! // significant 2 bits cleared and page_size is a power of 2
! mov(rscratch1, sp);
! sub(hdr, hdr, rscratch1);
! ands(hdr, hdr, aligned_mask - os::vm_page_size());
! // for recursive locking, the result is zero => save it in the displaced header
! // location (NULL in the displaced hdr location indicates recursive locking)
! str(hdr, Address(disp_hdr, 0));
! // otherwise we don't care about the result and handle locking via runtime call
! cbnz(hdr, slow_case);
! // done
! bind(done);
if (PrintBiasedLockingStatistics) {
lea(rscratch2, ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
addmw(Address(rscratch2, 0), 1, rscratch1);
}
return null_check_offset;
}
! void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
! assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
Label done;
if (UseBiasedLocking) {
// load object
ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
biased_locking_exit(obj, hdr, done);
}
! // load displaced header
! ldr(hdr, Address(disp_hdr, 0));
! // if the loaded hdr is NULL we had recursive locking
! // if we had recursive locking, we are done
! cbz(hdr, done);
if (!UseBiasedLocking) {
// load object
ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
}
verify_oop(obj);
! // test if object header is pointing to the displaced header, and if so, restore
! // the displaced header in the object - if the object header is not pointing to
! // the displaced header, get the object header instead
- // if the object header was not pointing to the displaced header,
- // we do unlocking via runtime call
- if (hdr_offset) {
- lea(rscratch1, Address(obj, hdr_offset));
- cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
} else {
! cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
}
- // done
- bind(done);
}
// Defines obj, preserves var_size_in_bytes
void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
tstw(hdr, JVM_ACC_IS_VALUE_BASED_CLASS);
br(Assembler::NE, slow_case);
}
if (UseBiasedLocking) {
! assert(temp != noreg, "should have temp register at this point");
! biased_locking_enter(disp_hdr, obj, hdr, temp, false, done, &slow_case);
}
! if (LockingMode == LM_LIGHTWEIGHT) {
! lightweight_lock(obj, hdr, temp, rscratch2, slow_case);
! } else {
! // Load object header
! ldr(hdr, Address(obj, hdr_offset));
! // and mark it as unlocked
! orr(hdr, hdr, markWord::unlocked_value);
! // save unlocked object header into the displaced header location on the stack
! str(hdr, Address(disp_hdr, 0));
! // test if object header is still the same (i.e. unlocked), and if so, store the
! // displaced header address in the object header - if it is not the same, get the
! // object header instead
! lea(rscratch2, Address(obj, hdr_offset));
! cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
! // if the object header was the same, we're done
! // if the object header was not the same, it is now in the hdr register
! // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
! //
! // 1) (hdr & aligned_mask) == 0
! // 2) sp <= hdr
! // 3) hdr <= sp + page_size
! //
! // these 3 tests can be done by evaluating the following expression:
! //
! // (hdr - sp) & (aligned_mask - page_size)
! //
! // assuming both the stack pointer and page_size have their least
! // significant 2 bits cleared and page_size is a power of 2
! mov(rscratch1, sp);
! sub(hdr, hdr, rscratch1);
! ands(hdr, hdr, aligned_mask - os::vm_page_size());
! // for recursive locking, the result is zero => save it in the displaced header
! // location (NULL in the displaced hdr location indicates recursive locking)
! str(hdr, Address(disp_hdr, 0));
! // otherwise we don't care about the result and handle locking via runtime call
+ cbnz(hdr, slow_case);
+ // done
+ bind(done);
+ }
if (PrintBiasedLockingStatistics) {
lea(rscratch2, ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
addmw(Address(rscratch2, 0), 1, rscratch1);
}
return null_check_offset;
}
! void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
! assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
Label done;
if (UseBiasedLocking) {
// load object
ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
biased_locking_exit(obj, hdr, done);
}
! if (LockingMode != LM_LIGHTWEIGHT) {
! // load displaced header
! ldr(hdr, Address(disp_hdr, 0));
! // if the loaded hdr is null we had recursive locking
! // if we had recursive locking, we are done
+ cbz(hdr, done);
+ }
+
if (!UseBiasedLocking) {
// load object
ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
}
verify_oop(obj);
!
! if (LockingMode == LM_LIGHTWEIGHT) {
! lightweight_unlock(obj, hdr, temp, rscratch2, slow_case);
} else {
! // test if object header is pointing to the displaced header, and if so, restore
+ // the displaced header in the object - if the object header is not pointing to
+ // the displaced header, get the object header instead
+ // if the object header was not pointing to the displaced header,
+ // we do unlocking via runtime call
+ if (hdr_offset) {
+ lea(rscratch1, Address(obj, hdr_offset));
+ cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
+ } else {
+ cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
+ }
+ // done
+ bind(done);
}
}
// Defines obj, preserves var_size_in_bytes
void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
}
}
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
assert_different_registers(obj, klass, len);
! if (UseBiasedLocking && !len->is_valid()) {
assert_different_registers(obj, klass, len, t1, t2);
ldr(t1, Address(klass, Klass::prototype_header_offset()));
} else {
// This assumes that all prototype bits fit in an int32_t
mov(t1, (int32_t)(intptr_t)markWord::prototype().value());
}
str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
! if (UseCompressedClassPointers) { // Take care not to kill klass
! encode_klass_not_null(t1, klass);
! strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
! } else {
! str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
}
if (len->is_valid()) {
strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
! } else if (UseCompressedClassPointers) {
store_klass_gap(obj, zr);
}
}
// preserves obj, destroys len_in_bytes
}
}
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
assert_different_registers(obj, klass, len);
! if (UseCompactObjectHeaders || (UseBiasedLocking && !len->is_valid())) {
assert_different_registers(obj, klass, len, t1, t2);
ldr(t1, Address(klass, Klass::prototype_header_offset()));
} else {
// This assumes that all prototype bits fit in an int32_t
mov(t1, (int32_t)(intptr_t)markWord::prototype().value());
}
str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
! if (!UseCompactObjectHeaders) {
! if (UseCompressedClassPointers) { // Take care not to kill klass
! encode_klass_not_null(t1, klass);
! strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
! } else {
+ str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ }
}
if (len->is_valid()) {
strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
! } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
store_klass_gap(obj, zr);
}
}
// preserves obj, destroys len_in_bytes
// len_in_bytes is positive and ptr sized
subs(len_in_bytes, len_in_bytes, hdr_size_in_bytes);
br(Assembler::EQ, done);
+ // Zero first 4 bytes, if start offset is not word aligned.
+ if (!is_aligned(hdr_size_in_bytes, BytesPerWord)) {
+ strw(zr, Address(obj, hdr_size_in_bytes));
+ hdr_size_in_bytes += BytesPerInt;
+ }
+
// zero_words() takes ptr in r10 and count in words in r11
mov(rscratch1, len_in_bytes);
lea(t1, Address(obj, hdr_size_in_bytes));
lsr(t2, rscratch1, LogBytesPerWord);
zero_words(t1, t2);
far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
}
verify_oop(obj);
}
! void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) {
assert_different_registers(obj, len, t1, t2, klass);
// determine alignment mask
assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
}
verify_oop(obj);
}
! void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
assert_different_registers(obj, len, t1, t2, klass);
// determine alignment mask
assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
cmp(len, rscratch1);
br(Assembler::HS, slow_case);
const Register arr_size = t2; // okay to be the same
// align object end
! mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
add(arr_size, arr_size, len, ext::uxtw, f);
andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
try_allocate(obj, arr_size, 0, t1, t2, slow_case);
initialize_header(obj, klass, len, t1, t2);
// clear rest of allocated space
! initialize_body(obj, arr_size, header_size * BytesPerWord, t1, t2);
membar(StoreStore);
if (CURRENT_ENV->dtrace_alloc_probes()) {
assert(obj == r0, "must be");
cmp(len, rscratch1);
br(Assembler::HS, slow_case);
const Register arr_size = t2; // okay to be the same
// align object end
! mov(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
add(arr_size, arr_size, len, ext::uxtw, f);
andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
try_allocate(obj, arr_size, 0, t1, t2, slow_case);
initialize_header(obj, klass, len, t1, t2);
// clear rest of allocated space
! initialize_body(obj, arr_size, base_offset_in_bytes, t1, t2);
membar(StoreStore);
if (CURRENT_ENV->dtrace_alloc_probes()) {
assert(obj == r0, "must be");
void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
verify_oop(receiver);
// explicit NULL check not needed since load from [klass_offset] causes a trap
// check against inline cache
! assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
!
cmp_klass(receiver, iCache, rscratch1);
}
void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
verify_oop(receiver);
// explicit NULL check not needed since load from [klass_offset] causes a trap
// check against inline cache
! if (UseCompactObjectHeaders) {
! assert(!MacroAssembler::needs_explicit_null_check(oopDesc::mark_offset_in_bytes()), "must add explicit null check");
+ } else {
+ assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
+ }
cmp_klass(receiver, iCache, rscratch1);
}
void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
< prev index next >