< prev index next > src/hotspot/cpu/x86/macroAssembler_x86.cpp
Print this page
#include "interpreter/interpreter.hpp"
#include "jvm.h"
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
#include "oops/accessDecorators.hpp"
+ #include "oops/compressedKlass.inline.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/klass.inline.hpp"
#include "prims/methodHandles.hpp"
#include "runtime/continuation.hpp"
#include "runtime/interfaceSupport.inline.hpp"
}
// Preserves the contents of address, destroys the contents length_in_bytes and temp.
void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
- assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
+ assert((offset_in_bytes & (BytesPerInt - 1)) == 0, "offset must be a multiple of BytesPerInt");
Label done;
testptr(length_in_bytes, length_in_bytes);
jcc(Assembler::zero, done);
+ // Emit single 32bit store to clear leading bytes, if necessary.
+ xorptr(temp, temp); // use _zero reg to clear memory (shorter code)
+ #ifdef _LP64
+ if (!is_aligned(offset_in_bytes, BytesPerWord)) {
+ movl(Address(address, offset_in_bytes), temp);
+ offset_in_bytes += BytesPerInt;
+ decrement(length_in_bytes, BytesPerInt);
+ }
+ assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
+ testptr(length_in_bytes, length_in_bytes);
+ jcc(Assembler::zero, done);
+ #endif
+
// initialize topmost word, divide index by 2, check if odd and test if zero
// note: for the remaining code to work, index must be a multiple of BytesPerWord
#ifdef ASSERT
{
Label L;
stop("length must be a multiple of BytesPerWord");
bind(L);
}
#endif
Register index = length_in_bytes;
- xorptr(temp, temp); // use _zero reg to clear memory (shorter code)
if (UseIncDec) {
shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set
} else {
shrptr(index, 2); // use 2 instructions to avoid partial flag stall
shrptr(index, 1);
movptr(holder, Address(method, Method::const_offset())); // ConstMethod*
movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
}
+ #ifdef _LP64
+ void MacroAssembler::load_nklass(Register dst, Register src) {
+ assert(UseCompressedClassPointers, "expect compressed class pointers");
+
+ if (!UseCompactObjectHeaders) {
+ movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+ return;
+ }
+
+ Label fast;
+ movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
+ testb(dst, markWord::monitor_value);
+ jccb(Assembler::zero, fast);
+
+ // Fetch displaced header
+ movq(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
+
+ bind(fast);
+ shrq(dst, markWord::klass_shift);
+ }
+ #endif
+
void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
assert_different_registers(src, tmp);
assert_different_registers(dst, tmp);
#ifdef _LP64
if (UseCompressedClassPointers) {
- movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+ load_nklass(dst, src);
decode_klass_not_null(dst, tmp);
} else
#endif
movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
+ assert(!UseCompactObjectHeaders, "not with compact headers");
assert_different_registers(src, tmp);
assert_different_registers(dst, tmp);
#ifdef _LP64
if (UseCompressedClassPointers) {
encode_klass_not_null(src, tmp);
movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
} else
#endif
- movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
+ movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
+ }
+
+ void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
+ #ifdef _LP64
+ if (UseCompactObjectHeaders) {
+ // NOTE: We need to deal with possible ObjectMonitor in object header.
+ // Eventually we might be able to do simple movl & cmpl like in
+ // the CCP path below.
+ load_nklass(tmp, obj);
+ cmpl(klass, tmp);
+ } else if (UseCompressedClassPointers) {
+ cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ } else
+ #endif
+ {
+ cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ }
+ }
+
+ void MacroAssembler::cmp_klass(Register src, Register dst, Register tmp1, Register tmp2) {
+ #ifdef _LP64
+ if (UseCompactObjectHeaders) {
+ // NOTE: We need to deal with possible ObjectMonitor in object header.
+ // Eventually we might be able to do simple movl & cmpl like in
+ // the CCP path below.
+ assert(tmp2 != noreg, "need tmp2");
+ assert_different_registers(src, dst, tmp1, tmp2);
+ load_nklass(tmp1, src);
+ load_nklass(tmp2, dst);
+ cmpl(tmp1, tmp2);
+ } else if (UseCompressedClassPointers) {
+ movl(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
+ cmpl(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
+ } else
+ #endif
+ {
+ movptr(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
+ cmpptr(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
+ }
}
void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
Register tmp1, Register thread_tmp) {
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
movq(dst, src);
}
}
}
+ MacroAssembler::KlassDecodeMode MacroAssembler::_klass_decode_mode = KlassDecodeNone;
+
+ // Returns a static string
+ const char* MacroAssembler::describe_klass_decode_mode(MacroAssembler::KlassDecodeMode mode) {
+ switch (mode) {
+ case KlassDecodeNone: return "none";
+ case KlassDecodeZero: return "zero";
+ case KlassDecodeXor: return "xor";
+ case KlassDecodeAdd: return "add";
+ default:
+ ShouldNotReachHere();
+ }
+ return NULL;
+ }
+
+ // Return the current narrow Klass pointer decode mode.
+ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() {
+ if (_klass_decode_mode == KlassDecodeNone) {
+ // First time initialization
+ assert(UseCompressedClassPointers, "not using compressed class pointers");
+ assert(Metaspace::initialized(), "metaspace not initialized yet");
+
+ _klass_decode_mode = klass_decode_mode_for_base(CompressedKlassPointers::base());
+ guarantee(_klass_decode_mode != KlassDecodeNone,
+ PTR_FORMAT " is not a valid encoding base on aarch64",
+ p2i(CompressedKlassPointers::base()));
+ log_info(metaspace)("klass decode mode initialized: %s", describe_klass_decode_mode(_klass_decode_mode));
+ }
+ return _klass_decode_mode;
+ }
+
+ // Given an arbitrary base address, return the KlassDecodeMode that would be used. Return KlassDecodeNone
+ // if base address is not valid for encoding.
+ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode_for_base(address base) {
+
+ const uint64_t base_u64 = (uint64_t) base;
+
+ if (base_u64 == 0) {
+ return KlassDecodeZero;
+ }
+
+ if ((base_u64 & (KlassEncodingMetaspaceMax - 1)) == 0) {
+ return KlassDecodeXor;
+ }
+
+ // Note that there is no point in optimizing for shift=3 since lilliput
+ // will use larger shifts
+
+ // The add+shift mode for decode_and_move_klass_not_null() requires the base to be
+ // shiftable-without-loss. So, this is the minimum restriction on x64 for a valid
+ // encoding base. This does not matter in reality since the shift values we use for
+ // Lilliput, while large, won't be larger than a page size. And the encoding base
+ // will be quite likely page aligned since it usually falls to the beginning of
+ // either CDS or CCS.
+ if ((base_u64 & (KlassAlignmentInBytes - 1)) == 0) {
+ return KlassDecodeAdd;
+ }
+
+ return KlassDecodeNone;
+ }
+
void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
assert_different_registers(r, tmp);
- if (CompressedKlassPointers::base() != nullptr) {
+ switch (klass_decode_mode()) {
+ case KlassDecodeZero: {
+ shrq(r, CompressedKlassPointers::shift());
+ break;
+ }
+ case KlassDecodeXor: {
+ mov64(tmp, (int64_t)CompressedKlassPointers::base());
+ xorq(r, tmp);
+ shrq(r, CompressedKlassPointers::shift());
+ break;
+ }
+ case KlassDecodeAdd: {
mov64(tmp, (int64_t)CompressedKlassPointers::base());
subq(r, tmp);
+ shrq(r, CompressedKlassPointers::shift());
+ break;
}
- if (CompressedKlassPointers::shift() != 0) {
- assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
- shrq(r, LogKlassAlignmentInBytes);
+ default:
+ ShouldNotReachHere();
}
}
void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
assert_different_registers(src, dst);
- if (CompressedKlassPointers::base() != nullptr) {
+ switch (klass_decode_mode()) {
+ case KlassDecodeZero: {
+ movptr(dst, src);
+ shrq(dst, CompressedKlassPointers::shift());
+ break;
+ }
+ case KlassDecodeXor: {
+ mov64(dst, (int64_t)CompressedKlassPointers::base());
+ xorq(dst, src);
+ shrq(dst, CompressedKlassPointers::shift());
+ break;
+ }
+ case KlassDecodeAdd: {
mov64(dst, -(int64_t)CompressedKlassPointers::base());
addq(dst, src);
- } else {
- movptr(dst, src);
+ shrq(dst, CompressedKlassPointers::shift());
+ break;
}
- if (CompressedKlassPointers::shift() != 0) {
- assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
- shrq(dst, LogKlassAlignmentInBytes);
+ default:
+ ShouldNotReachHere();
}
}
void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
assert_different_registers(r, tmp);
- // Note: it will change flags
- assert(UseCompressedClassPointers, "should only be used for compressed headers");
- // Cannot assert, unverified entry point counts instructions (see .ad file)
- // vtableStubs also counts instructions in pd_code_size_limit.
- // Also do not verify_oop as this is called by verify_oop.
- if (CompressedKlassPointers::shift() != 0) {
- assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
- shlq(r, LogKlassAlignmentInBytes);
+ const uint64_t base_u64 = (uint64_t)CompressedKlassPointers::base();
+ switch (klass_decode_mode()) {
+ case KlassDecodeZero: {
+ shlq(r, CompressedKlassPointers::shift());
+ break;
}
- if (CompressedKlassPointers::base() != nullptr) {
- mov64(tmp, (int64_t)CompressedKlassPointers::base());
+ case KlassDecodeXor: {
+ assert((base_u64 & (KlassEncodingMetaspaceMax - 1)) == 0,
+ "base " UINT64_FORMAT_X " invalid for xor mode", base_u64); // should have been handled at VM init.
+ shlq(r, CompressedKlassPointers::shift());
+ mov64(tmp, base_u64);
+ xorq(r, tmp);
+ break;
+ }
+ case KlassDecodeAdd: {
+ shlq(r, CompressedKlassPointers::shift());
+ mov64(tmp, base_u64);
addq(r, tmp);
+ break;
+ }
+ default:
+ ShouldNotReachHere();
}
}
void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
assert_different_registers(src, dst);
- // Note: it will change flags
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
- // Cannot assert, unverified entry point counts instructions (see .ad file)
+ // Note: Cannot assert, unverified entry point counts instructions (see .ad file)
// vtableStubs also counts instructions in pd_code_size_limit.
// Also do not verify_oop as this is called by verify_oop.
- if (CompressedKlassPointers::base() == nullptr &&
- CompressedKlassPointers::shift() == 0) {
- // The best case scenario is that there is no base or shift. Then it is already
- // a pointer that needs nothing but a register rename.
- movl(dst, src);
- } else {
- if (CompressedKlassPointers::base() != nullptr) {
- mov64(dst, (int64_t)CompressedKlassPointers::base());
- } else {
- xorq(dst, dst);
- }
- if (CompressedKlassPointers::shift() != 0) {
- assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
- assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
- leaq(dst, Address(dst, src, Address::times_8, 0));
- } else {
- addq(dst, src);
- }
+ const uint64_t base_u64 = (uint64_t)CompressedKlassPointers::base();
+
+ switch (klass_decode_mode()) {
+ case KlassDecodeZero: {
+ movq(dst, src);
+ shlq(dst, CompressedKlassPointers::shift());
+ break;
+ }
+ case KlassDecodeXor: {
+ assert((base_u64 & (KlassEncodingMetaspaceMax - 1)) == 0,
+ "base " UINT64_FORMAT_X " invalid for xor mode", base_u64); // should have been handled at VM init.
+ const uint64_t base_right_shifted = base_u64 >> CompressedKlassPointers::shift();
+ mov64(dst, base_right_shifted);
+ xorq(dst, src);
+ shlq(dst, CompressedKlassPointers::shift());
+ break;
+ }
+ case KlassDecodeAdd: {
+ assert((base_u64 & (KlassAlignmentInBytes - 1)) == 0,
+ "base " UINT64_FORMAT_X " invalid for add mode", base_u64); // should have been handled at VM init.
+ const uint64_t base_right_shifted = base_u64 >> CompressedKlassPointers::shift();
+ mov64(dst, base_right_shifted);
+ addq(dst, src);
+ shlq(dst, CompressedKlassPointers::shift());
+ break;
+ }
+ default:
+ ShouldNotReachHere();
}
}
void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
assert (UseCompressedOops, "should only be used for compressed headers");
< prev index next >