< prev index next > src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
Print this page
/*
- * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
cset(result, NE); // Not equal or unordered
cneg(result, result, LO); // Less than
}
}
- int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register scratch, Label& slow_case) {
+ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
- assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
+ assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
Label done;
int null_check_offset = -1;
verify_oop(obj);
tstw(hdr, JVM_ACC_IS_VALUE_BASED_CLASS);
br(Assembler::NE, slow_case);
}
if (UseBiasedLocking) {
- assert(scratch != noreg, "should have scratch register at this point");
- biased_locking_enter(disp_hdr, obj, hdr, scratch, false, done, &slow_case);
+ assert(temp != noreg, "should have temp register at this point");
+ biased_locking_enter(disp_hdr, obj, hdr, temp, false, done, &slow_case);
}
- // Load object header
- ldr(hdr, Address(obj, hdr_offset));
- // and mark it as unlocked
- orr(hdr, hdr, markWord::unlocked_value);
- // save unlocked object header into the displaced header location on the stack
- str(hdr, Address(disp_hdr, 0));
- // test if object header is still the same (i.e. unlocked), and if so, store the
- // displaced header address in the object header - if it is not the same, get the
- // object header instead
- lea(rscratch2, Address(obj, hdr_offset));
- cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
- // if the object header was the same, we're done
- // if the object header was not the same, it is now in the hdr register
- // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
- //
- // 1) (hdr & aligned_mask) == 0
- // 2) sp <= hdr
- // 3) hdr <= sp + page_size
- //
- // these 3 tests can be done by evaluating the following expression:
- //
- // (hdr - sp) & (aligned_mask - page_size)
- //
- // assuming both the stack pointer and page_size have their least
- // significant 2 bits cleared and page_size is a power of 2
- mov(rscratch1, sp);
- sub(hdr, hdr, rscratch1);
- ands(hdr, hdr, aligned_mask - os::vm_page_size());
- // for recursive locking, the result is zero => save it in the displaced header
- // location (NULL in the displaced hdr location indicates recursive locking)
- str(hdr, Address(disp_hdr, 0));
- // otherwise we don't care about the result and handle locking via runtime call
- cbnz(hdr, slow_case);
- // done
- bind(done);
+ if (LockingMode == LM_LIGHTWEIGHT) {
+ lightweight_lock(obj, hdr, temp, rscratch2, slow_case);
+ } else {
+ // Load object header
+ ldr(hdr, Address(obj, hdr_offset));
+ // and mark it as unlocked
+ orr(hdr, hdr, markWord::unlocked_value);
+ // save unlocked object header into the displaced header location on the stack
+ str(hdr, Address(disp_hdr, 0));
+ // test if object header is still the same (i.e. unlocked), and if so, store the
+ // displaced header address in the object header - if it is not the same, get the
+ // object header instead
+ lea(rscratch2, Address(obj, hdr_offset));
+ cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
+ // if the object header was the same, we're done
+ // if the object header was not the same, it is now in the hdr register
+ // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
+ //
+ // 1) (hdr & aligned_mask) == 0
+ // 2) sp <= hdr
+ // 3) hdr <= sp + page_size
+ //
+ // these 3 tests can be done by evaluating the following expression:
+ //
+ // (hdr - sp) & (aligned_mask - page_size)
+ //
+ // assuming both the stack pointer and page_size have their least
+ // significant 2 bits cleared and page_size is a power of 2
+ mov(rscratch1, sp);
+ sub(hdr, hdr, rscratch1);
+ ands(hdr, hdr, aligned_mask - os::vm_page_size());
+ // for recursive locking, the result is zero => save it in the displaced header
+ // location (NULL in the displaced hdr location indicates recursive locking)
+ str(hdr, Address(disp_hdr, 0));
+ // otherwise we don't care about the result and handle locking via runtime call
+ cbnz(hdr, slow_case);
+ // done
+ bind(done);
+ }
if (PrintBiasedLockingStatistics) {
lea(rscratch2, ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
addmw(Address(rscratch2, 0), 1, rscratch1);
}
return null_check_offset;
}
- void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
+ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
- assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
+ assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
Label done;
if (UseBiasedLocking) {
// load object
ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
biased_locking_exit(obj, hdr, done);
}
- // load displaced header
- ldr(hdr, Address(disp_hdr, 0));
- // if the loaded hdr is NULL we had recursive locking
- // if we had recursive locking, we are done
- cbz(hdr, done);
+ if (LockingMode != LM_LIGHTWEIGHT) {
+ // load displaced header
+ ldr(hdr, Address(disp_hdr, 0));
+ // if the loaded hdr is null we had recursive locking
+ // if we had recursive locking, we are done
+ cbz(hdr, done);
+ }
+
if (!UseBiasedLocking) {
// load object
ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
}
verify_oop(obj);
- // test if object header is pointing to the displaced header, and if so, restore
- // the displaced header in the object - if the object header is not pointing to
- // the displaced header, get the object header instead
- // if the object header was not pointing to the displaced header,
- // we do unlocking via runtime call
- if (hdr_offset) {
- lea(rscratch1, Address(obj, hdr_offset));
- cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
+
+ if (LockingMode == LM_LIGHTWEIGHT) {
+ lightweight_unlock(obj, hdr, temp, rscratch2, slow_case);
} else {
- cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
+ // test if object header is pointing to the displaced header, and if so, restore
+ // the displaced header in the object - if the object header is not pointing to
+ // the displaced header, get the object header instead
+ // if the object header was not pointing to the displaced header,
+ // we do unlocking via runtime call
+ if (hdr_offset) {
+ lea(rscratch1, Address(obj, hdr_offset));
+ cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
+ } else {
+ cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
+ }
+ // done
+ bind(done);
}
- // done
- bind(done);
}
// Defines obj, preserves var_size_in_bytes
void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
}
}
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
assert_different_registers(obj, klass, len);
- if (UseBiasedLocking && !len->is_valid()) {
+ if (UseCompactObjectHeaders || (UseBiasedLocking && !len->is_valid())) {
assert_different_registers(obj, klass, len, t1, t2);
ldr(t1, Address(klass, Klass::prototype_header_offset()));
} else {
// This assumes that all prototype bits fit in an int32_t
mov(t1, (int32_t)(intptr_t)markWord::prototype().value());
}
str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
- if (UseCompressedClassPointers) { // Take care not to kill klass
- encode_klass_not_null(t1, klass);
- strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
- } else {
- str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ if (!UseCompactObjectHeaders) {
+ if (UseCompressedClassPointers) { // Take care not to kill klass
+ encode_klass_not_null(t1, klass);
+ strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
+ } else {
+ str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ }
}
if (len->is_valid()) {
strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
- } else if (UseCompressedClassPointers) {
+ } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
store_klass_gap(obj, zr);
}
}
// preserves obj, destroys len_in_bytes
// len_in_bytes is positive and ptr sized
subs(len_in_bytes, len_in_bytes, hdr_size_in_bytes);
br(Assembler::EQ, done);
+ // Zero first 4 bytes, if start offset is not word aligned.
+ if (!is_aligned(hdr_size_in_bytes, BytesPerWord)) {
+ strw(zr, Address(obj, hdr_size_in_bytes));
+ hdr_size_in_bytes += BytesPerInt;
+ }
+
// zero_words() takes ptr in r10 and count in words in r11
mov(rscratch1, len_in_bytes);
lea(t1, Address(obj, hdr_size_in_bytes));
lsr(t2, rscratch1, LogBytesPerWord);
zero_words(t1, t2);
far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
}
verify_oop(obj);
}
- void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) {
+ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
assert_different_registers(obj, len, t1, t2, klass);
// determine alignment mask
assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
cmp(len, rscratch1);
br(Assembler::HS, slow_case);
const Register arr_size = t2; // okay to be the same
// align object end
- mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+ mov(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
add(arr_size, arr_size, len, ext::uxtw, f);
andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
try_allocate(obj, arr_size, 0, t1, t2, slow_case);
initialize_header(obj, klass, len, t1, t2);
// clear rest of allocated space
- initialize_body(obj, arr_size, header_size * BytesPerWord, t1, t2);
+ initialize_body(obj, arr_size, base_offset_in_bytes, t1, t2);
membar(StoreStore);
if (CURRENT_ENV->dtrace_alloc_probes()) {
assert(obj == r0, "must be");
void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
verify_oop(receiver);
// explicit NULL check not needed since load from [klass_offset] causes a trap
// check against inline cache
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
-
+ if (UseCompactObjectHeaders) {
+ assert(!MacroAssembler::needs_explicit_null_check(oopDesc::mark_offset_in_bytes()), "must add explicit null check");
+ } else {
+ assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
+ }
cmp_klass(receiver, iCache, rscratch1);
}
void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
< prev index next >