< prev index next >

src/hotspot/cpu/x86/macroAssembler_x86.cpp

Print this page
@@ -36,10 +36,11 @@
  #include "interpreter/interpreter.hpp"
  #include "jvm.h"
  #include "memory/resourceArea.hpp"
  #include "memory/universe.hpp"
  #include "oops/accessDecorators.hpp"
+ #include "oops/compressedKlass.inline.hpp"
  #include "oops/compressedOops.inline.hpp"
  #include "oops/klass.inline.hpp"
  #include "prims/methodHandles.hpp"
  #include "runtime/continuation.hpp"
  #include "runtime/interfaceSupport.inline.hpp"

@@ -4167,16 +4168,29 @@
  }
  
  // Preserves the contents of address, destroys the contents length_in_bytes and temp.
  void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
    assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
-   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
+   assert((offset_in_bytes & (BytesPerInt - 1)) == 0, "offset must be a multiple of BytesPerInt");
    Label done;
  
    testptr(length_in_bytes, length_in_bytes);
    jcc(Assembler::zero, done);
  
+   // Emit single 32bit store to clear leading bytes, if necessary.
+   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
+ #ifdef _LP64
+   if (!is_aligned(offset_in_bytes, BytesPerWord)) {
+     movl(Address(address, offset_in_bytes), temp);
+     offset_in_bytes += BytesPerInt;
+     decrement(length_in_bytes, BytesPerInt);
+   }
+   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
+   testptr(length_in_bytes, length_in_bytes);
+   jcc(Assembler::zero, done);
+ #endif
+ 
    // initialize topmost word, divide index by 2, check if odd and test if zero
    // note: for the remaining code to work, index must be a multiple of BytesPerWord
  #ifdef ASSERT
    {
      Label L;

@@ -4185,11 +4199,10 @@
      stop("length must be a multiple of BytesPerWord");
      bind(L);
    }
  #endif
    Register index = length_in_bytes;
-   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
    if (UseIncDec) {
      shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
    } else {
      shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
      shrptr(index, 1);

@@ -5117,32 +5130,94 @@
    movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
    movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
    movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
  }
  
+ #ifdef _LP64
+ void MacroAssembler::load_nklass(Register dst, Register src) {
+   assert(UseCompressedClassPointers, "expect compressed class pointers");
+ 
+   if (!UseCompactObjectHeaders) {
+     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+     return;
+   }
+ 
+   Label fast;
+   movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
+   testb(dst, markWord::monitor_value);
+   jccb(Assembler::zero, fast);
+ 
+   // Fetch displaced header
+   movq(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
+ 
+   bind(fast);
+   shrq(dst, markWord::klass_shift);
+ }
+ #endif
+ 
  void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
    assert_different_registers(src, tmp);
    assert_different_registers(dst, tmp);
  #ifdef _LP64
    if (UseCompressedClassPointers) {
-     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+     load_nklass(dst, src);
      decode_klass_not_null(dst, tmp);
    } else
  #endif
      movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
  }
  
  void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
+   assert(!UseCompactObjectHeaders, "not with compact headers");
    assert_different_registers(src, tmp);
    assert_different_registers(dst, tmp);
  #ifdef _LP64
    if (UseCompressedClassPointers) {
      encode_klass_not_null(src, tmp);
      movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
    } else
  #endif
-     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
+    movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
+ }
+ 
+ void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
+ #ifdef _LP64
+   if (UseCompactObjectHeaders) {
+     // NOTE: We need to deal with possible ObjectMonitor in object header.
+     // Eventually we might be able to do simple movl & cmpl like in
+     // the CCP path below.
+     load_nklass(tmp, obj);
+     cmpl(klass, tmp);
+   } else if (UseCompressedClassPointers) {
+     cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+   } else
+ #endif
+   {
+     cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+   }
+ }
+ 
+ void MacroAssembler::cmp_klass(Register src, Register dst, Register tmp1, Register tmp2) {
+ #ifdef _LP64
+   if (UseCompactObjectHeaders) {
+     // NOTE: We need to deal with possible ObjectMonitor in object header.
+     // Eventually we might be able to do simple movl & cmpl like in
+     // the CCP path below.
+     assert(tmp2 != noreg, "need tmp2");
+     assert_different_registers(src, dst, tmp1, tmp2);
+     load_nklass(tmp1, src);
+     load_nklass(tmp2, dst);
+     cmpl(tmp1, tmp2);
+   } else if (UseCompressedClassPointers) {
+     movl(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
+     cmpl(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
+   } else
+ #endif
+   {
+     movptr(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
+     cmpptr(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
+   }
  }
  
  void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
                                      Register tmp1, Register thread_tmp) {
    BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();

@@ -5345,79 +5420,181 @@
        movq(dst, src);
      }
    }
  }
  
+ MacroAssembler::KlassDecodeMode MacroAssembler::_klass_decode_mode = KlassDecodeNone;
+ 
+ // Returns a static string
+ const char* MacroAssembler::describe_klass_decode_mode(MacroAssembler::KlassDecodeMode mode) {
+   switch (mode) {
+   case KlassDecodeNone: return "none";
+   case KlassDecodeZero: return "zero";
+   case KlassDecodeXor:  return "xor";
+   case KlassDecodeAdd:  return "add";
+   default:
+     ShouldNotReachHere();
+   }
+   return NULL;
+ }
+ 
+ // Return the current narrow Klass pointer decode mode.
+ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() {
+   if (_klass_decode_mode == KlassDecodeNone) {
+     // First time initialization
+     assert(UseCompressedClassPointers, "not using compressed class pointers");
+     assert(Metaspace::initialized(), "metaspace not initialized yet");
+ 
+     _klass_decode_mode = klass_decode_mode_for_base(CompressedKlassPointers::base());
+     guarantee(_klass_decode_mode != KlassDecodeNone,
+               PTR_FORMAT " is not a valid encoding base on aarch64",
+               p2i(CompressedKlassPointers::base()));
+     log_info(metaspace)("klass decode mode initialized: %s", describe_klass_decode_mode(_klass_decode_mode));
+   }
+   return _klass_decode_mode;
+ }
+ 
+ // Given an arbitrary base address, return the KlassDecodeMode that would be used. Return KlassDecodeNone
+ // if base address is not valid for encoding.
+ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode_for_base(address base) {
+ 
+   const uint64_t base_u64 = (uint64_t) base;
+ 
+   if (base_u64 == 0) {
+     return KlassDecodeZero;
+   }
+ 
+   if ((base_u64 & (KlassEncodingMetaspaceMax - 1)) == 0) {
+     return KlassDecodeXor;
+   }
+ 
+   // Note that there is no point in optimizing for shift=3 since lilliput
+   // will use larger shifts
+ 
+   // The add+shift mode for decode_and_move_klass_not_null() requires the base to be
+   //  shiftable-without-loss. So, this is the minimum restriction on x64 for a valid
+   //  encoding base. This does not matter in reality since the shift values we use for
+   //  Lilliput, while large, won't be larger than a page size. And the encoding base
+   //  will be quite likely page aligned since it usually falls to the beginning of
+   //  either CDS or CCS.
+   if ((base_u64 & (KlassAlignmentInBytes - 1)) == 0) {
+     return KlassDecodeAdd;
+   }
+ 
+   return KlassDecodeNone;
+ }
+ 
  void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
    assert_different_registers(r, tmp);
-   if (CompressedKlassPointers::base() != nullptr) {
+   switch (klass_decode_mode()) {
+   case KlassDecodeZero: {
+     shrq(r, CompressedKlassPointers::shift());
+     break;
+   }
+   case KlassDecodeXor: {
+     mov64(tmp, (int64_t)CompressedKlassPointers::base());
+     xorq(r, tmp);
+     shrq(r, CompressedKlassPointers::shift());
+     break;
+   }
+   case KlassDecodeAdd: {
      mov64(tmp, (int64_t)CompressedKlassPointers::base());
      subq(r, tmp);
+     shrq(r, CompressedKlassPointers::shift());
+     break;
    }
-   if (CompressedKlassPointers::shift() != 0) {
-     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
-     shrq(r, LogKlassAlignmentInBytes);
+   default:
+     ShouldNotReachHere();
    }
  }
  
  void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
    assert_different_registers(src, dst);
-   if (CompressedKlassPointers::base() != nullptr) {
+   switch (klass_decode_mode()) {
+   case KlassDecodeZero: {
+     movptr(dst, src);
+     shrq(dst, CompressedKlassPointers::shift());
+     break;
+   }
+   case KlassDecodeXor: {
+     mov64(dst, (int64_t)CompressedKlassPointers::base());
+     xorq(dst, src);
+     shrq(dst, CompressedKlassPointers::shift());
+     break;
+   }
+   case KlassDecodeAdd: {
      mov64(dst, -(int64_t)CompressedKlassPointers::base());
      addq(dst, src);
-   } else {
-     movptr(dst, src);
+     shrq(dst, CompressedKlassPointers::shift());
+     break;
    }
-   if (CompressedKlassPointers::shift() != 0) {
-     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
-     shrq(dst, LogKlassAlignmentInBytes);
+   default:
+     ShouldNotReachHere();
    }
  }
  
  void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
    assert_different_registers(r, tmp);
-   // Note: it will change flags
-   assert(UseCompressedClassPointers, "should only be used for compressed headers");
-   // Cannot assert, unverified entry point counts instructions (see .ad file)
-   // vtableStubs also counts instructions in pd_code_size_limit.
-   // Also do not verify_oop as this is called by verify_oop.
-   if (CompressedKlassPointers::shift() != 0) {
-     assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
-     shlq(r, LogKlassAlignmentInBytes);
+   const uint64_t base_u64 = (uint64_t)CompressedKlassPointers::base();
+   switch (klass_decode_mode()) {
+   case KlassDecodeZero: {
+     shlq(r, CompressedKlassPointers::shift());
+     break;
    }
-   if (CompressedKlassPointers::base() != nullptr) {
-     mov64(tmp, (int64_t)CompressedKlassPointers::base());
+   case KlassDecodeXor: {
+     assert((base_u64 & (KlassEncodingMetaspaceMax - 1)) == 0,
+            "base " UINT64_FORMAT_X " invalid for xor mode", base_u64); // should have been handled at VM init.
+     shlq(r, CompressedKlassPointers::shift());
+     mov64(tmp, base_u64);
+     xorq(r, tmp);
+     break;
+   }
+   case KlassDecodeAdd: {
+     shlq(r, CompressedKlassPointers::shift());
+     mov64(tmp, base_u64);
      addq(r, tmp);
+     break;
+   }
+   default:
+     ShouldNotReachHere();
    }
  }
  
  void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
    assert_different_registers(src, dst);
-   // Note: it will change flags
-   assert (UseCompressedClassPointers, "should only be used for compressed headers");
-   // Cannot assert, unverified entry point counts instructions (see .ad file)
+   // Note: Cannot assert, unverified entry point counts instructions (see .ad file)
    // vtableStubs also counts instructions in pd_code_size_limit.
    // Also do not verify_oop as this is called by verify_oop.
  
-   if (CompressedKlassPointers::base() == nullptr &&
-       CompressedKlassPointers::shift() == 0) {
-     // The best case scenario is that there is no base or shift. Then it is already
-     // a pointer that needs nothing but a register rename.
-     movl(dst, src);
-   } else {
-     if (CompressedKlassPointers::base() != nullptr) {
-       mov64(dst, (int64_t)CompressedKlassPointers::base());
-     } else {
-       xorq(dst, dst);
-     }
-     if (CompressedKlassPointers::shift() != 0) {
-       assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
-       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
-       leaq(dst, Address(dst, src, Address::times_8, 0));
-     } else {
-       addq(dst, src);
-     }
+   const uint64_t base_u64 = (uint64_t)CompressedKlassPointers::base();
+ 
+   switch (klass_decode_mode()) {
+   case KlassDecodeZero: {
+     movq(dst, src);
+     shlq(dst, CompressedKlassPointers::shift());
+     break;
+   }
+   case KlassDecodeXor: {
+     assert((base_u64 & (KlassEncodingMetaspaceMax - 1)) == 0,
+            "base " UINT64_FORMAT_X " invalid for xor mode", base_u64); // should have been handled at VM init.
+     const uint64_t base_right_shifted = base_u64 >> CompressedKlassPointers::shift();
+     mov64(dst, base_right_shifted);
+     xorq(dst, src);
+     shlq(dst, CompressedKlassPointers::shift());
+     break;
+   }
+   case KlassDecodeAdd: {
+     assert((base_u64 & (KlassAlignmentInBytes - 1)) == 0,
+            "base " UINT64_FORMAT_X " invalid for add mode", base_u64); // should have been handled at VM init.
+     const uint64_t base_right_shifted = base_u64 >> CompressedKlassPointers::shift();
+     mov64(dst, base_right_shifted);
+     addq(dst, src);
+     shlq(dst, CompressedKlassPointers::shift());
+     break;
+   }
+   default:
+     ShouldNotReachHere();
    }
  }
  
  void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
    assert (UseCompressedOops, "should only be used for compressed headers");
< prev index next >