< prev index next >

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

Print this page
*** 43,10 ***
--- 43,11 ---
  #include "jvm.h"
  #include "memory/resourceArea.hpp"
  #include "memory/universe.hpp"
  #include "nativeInst_aarch64.hpp"
  #include "oops/accessDecorators.hpp"
+ #include "oops/compressedKlass.inline.hpp"
  #include "oops/compressedOops.inline.hpp"
  #include "oops/klass.inline.hpp"
  #include "runtime/continuation.hpp"
  #include "runtime/icache.hpp"
  #include "runtime/interfaceSupport.inline.hpp"

*** 4250,13 ***
    ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
    ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
    ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
  }
  
  void MacroAssembler::load_klass(Register dst, Register src) {
    if (UseCompressedClassPointers) {
!     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
      decode_klass_not_null(dst);
    } else {
      ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
    }
  }
--- 4251,41 ---
    ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
    ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
    ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
  }
  
+ // Loads the obj's Klass* into dst.
+ // Preserves all registers (incl src, rscratch1 and rscratch2).
+ void MacroAssembler::load_nklass(Register dst, Register src) {
+   assert(UseCompressedClassPointers, "expects UseCompressedClassPointers");
+ 
+   if (!UseCompactObjectHeaders) {
+     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+     return;
+   }
+ 
+   Label fast;
+ 
+   // Check if we can take the (common) fast path, if obj is unlocked.
+   ldr(dst, Address(src, oopDesc::mark_offset_in_bytes()));
+   tbz(dst, exact_log2(markWord::monitor_value), fast);
+ 
+   // Fetch displaced header
+   ldr(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
+ 
+   // Fast-path: shift and decode Klass*.
+   bind(fast);
+   lsr(dst, dst, markWord::klass_shift);
+ }
+ 
  void MacroAssembler::load_klass(Register dst, Register src) {
    if (UseCompressedClassPointers) {
!     if (UseCompactObjectHeaders) {
+       load_nklass(dst, src);
+     } else {
+       ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+     }
      decode_klass_not_null(dst);
    } else {
      ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
    }
  }

*** 4295,12 ***
    ldr(dst, Address(dst, mirror_offset));
    resolve_oop_handle(dst, tmp1, tmp2);
  }
  
  void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
    if (UseCompressedClassPointers) {
!     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
      if (CompressedKlassPointers::base() == NULL) {
        cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
        return;
      } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
                 && CompressedKlassPointers::shift() == 0) {
--- 4324,17 ---
    ldr(dst, Address(dst, mirror_offset));
    resolve_oop_handle(dst, tmp1, tmp2);
  }
  
  void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
+   assert_different_registers(oop, trial_klass, tmp);
    if (UseCompressedClassPointers) {
!     if (UseCompactObjectHeaders) {
+       load_nklass(tmp, oop);
+     } else {
+       ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
+     }
      if (CompressedKlassPointers::base() == NULL) {
        cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
        return;
      } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
                 && CompressedKlassPointers::shift() == 0) {

*** 4465,67 ***
    }
  }
  
  MacroAssembler::KlassDecodeMode MacroAssembler::_klass_decode_mode(KlassDecodeNone);
  
  MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() {
!   assert(UseCompressedClassPointers, "not using compressed class pointers");
!   assert(Metaspace::initialized(), "metaspace not initialized yet");
  
!   if (_klass_decode_mode != KlassDecodeNone) {
!     return _klass_decode_mode;
    }
  
!   assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift()
!          || 0 == CompressedKlassPointers::shift(), "decode alg wrong");
  
!   if (CompressedKlassPointers::base() == NULL) {
!     return (_klass_decode_mode = KlassDecodeZero);
    }
  
!   if (operand_valid_for_logical_immediate(
!         /*is32*/false, (uint64_t)CompressedKlassPointers::base())) {
!     const uint64_t range_mask =
-       (1ULL << log2i(CompressedKlassPointers::range())) - 1;
-     if (((uint64_t)CompressedKlassPointers::base() & range_mask) == 0) {
-       return (_klass_decode_mode = KlassDecodeXor);
-     }
    }
  
!   const uint64_t shifted_base =
!     (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
!   guarantee((shifted_base & 0xffff0000ffffffff) == 0,
!             "compressed class base bad alignment");
  
!   return (_klass_decode_mode = KlassDecodeMovk);
  }
  
  void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
    switch (klass_decode_mode()) {
    case KlassDecodeZero:
!     if (CompressedKlassPointers::shift() != 0) {
-       lsr(dst, src, LogKlassAlignmentInBytes);
-     } else {
-       if (dst != src) mov(dst, src);
-     }
      break;
  
    case KlassDecodeXor:
!     if (CompressedKlassPointers::shift() != 0) {
!       eor(dst, src, (uint64_t)CompressedKlassPointers::base());
-       lsr(dst, dst, LogKlassAlignmentInBytes);
-     } else {
-       eor(dst, src, (uint64_t)CompressedKlassPointers::base());
-     }
      break;
  
    case KlassDecodeMovk:
!     if (CompressedKlassPointers::shift() != 0) {
-       ubfx(dst, src, LogKlassAlignmentInBytes, 32);
-     } else {
-       movw(dst, src);
-     }
      break;
  
    case KlassDecodeNone:
      ShouldNotReachHere();
      break;
--- 4499,78 ---
    }
  }
  
  MacroAssembler::KlassDecodeMode MacroAssembler::_klass_decode_mode(KlassDecodeNone);
  
+ // Returns a static string
+ const char* MacroAssembler::describe_klass_decode_mode(MacroAssembler::KlassDecodeMode mode) {
+   switch (mode) {
+   case KlassDecodeNone: return "none";
+   case KlassDecodeZero: return "zero";
+   case KlassDecodeXor:  return "xor";
+   case KlassDecodeMovk: return "movk";
+   default:
+     ShouldNotReachHere();
+   }
+   return NULL;
+ }
+ 
+ // Return the current narrow Klass pointer decode mode.
  MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() {
!   if (_klass_decode_mode == KlassDecodeNone) {
!     // First time initialization
+     assert(UseCompressedClassPointers, "not using compressed class pointers");
+     assert(Metaspace::initialized(), "metaspace not initialized yet");
  
!     _klass_decode_mode = klass_decode_mode_for_base(CompressedKlassPointers::base());
!     guarantee(_klass_decode_mode != KlassDecodeNone,
+               PTR_FORMAT " is not a valid encoding base on aarch64",
+               p2i(CompressedKlassPointers::base()));
+     log_info(metaspace)("klass decode mode initialized: %s", describe_klass_decode_mode(_klass_decode_mode));
    }
+   return _klass_decode_mode;
+ }
  
! // Given an arbitrary base address, return the KlassDecodeMode that would be used. Return KlassDecodeNone
! // if base address is not valid for encoding.
+ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode_for_base(address base) {
+   assert(CompressedKlassPointers::shift() != 0, "not lilliput?");
  
!   const uint64_t base_u64 = (uint64_t) base;
! 
+   if (base_u64 == 0) {
+     return KlassDecodeZero;
    }
  
!   if (operand_valid_for_logical_immediate(false, base_u64) &&
!       ((base_u64 & (KlassEncodingMetaspaceMax - 1)) == 0)) {
!     return KlassDecodeXor;
    }
  
!   const uint64_t shifted_base = base_u64 >> CompressedKlassPointers::shift();
!   if ((shifted_base & 0xffff0000ffffffff) == 0) {
!     return KlassDecodeMovk;
!   }
  
!   return KlassDecodeNone;
  }
  
  void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
+   assert (UseCompressedClassPointers, "should only be used for compressed headers");
+   assert(CompressedKlassPointers::shift() != 0, "not lilliput?");
    switch (klass_decode_mode()) {
    case KlassDecodeZero:
!     lsr(dst, src, LogKlassAlignmentInBytes);
      break;
  
    case KlassDecodeXor:
!     eor(dst, src, (uint64_t)CompressedKlassPointers::base());
!     lsr(dst, dst, LogKlassAlignmentInBytes);
      break;
  
    case KlassDecodeMovk:
!     ubfx(dst, src, LogKlassAlignmentInBytes, MaxNarrowKlassPointerBits);
      break;
  
    case KlassDecodeNone:
      ShouldNotReachHere();
      break;

*** 4537,39 ***
  }
  
  void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
    assert (UseCompressedClassPointers, "should only be used for compressed headers");
  
    switch (klass_decode_mode()) {
    case KlassDecodeZero:
!     if (CompressedKlassPointers::shift() != 0) {
-       lsl(dst, src, LogKlassAlignmentInBytes);
-     } else {
-       if (dst != src) mov(dst, src);
-     }
      break;
  
    case KlassDecodeXor:
!     if (CompressedKlassPointers::shift() != 0) {
!       lsl(dst, src, LogKlassAlignmentInBytes);
-       eor(dst, dst, (uint64_t)CompressedKlassPointers::base());
-     } else {
-       eor(dst, src, (uint64_t)CompressedKlassPointers::base());
-     }
      break;
  
    case KlassDecodeMovk: {
      const uint64_t shifted_base =
        (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
  
      if (dst != src) movw(dst, src);
      movk(dst, shifted_base >> 32, 32);
! 
-     if (CompressedKlassPointers::shift() != 0) {
-       lsl(dst, dst, LogKlassAlignmentInBytes);
-     }
- 
      break;
    }
  
    case KlassDecodeNone:
      ShouldNotReachHere();
--- 4582,32 ---
  }
  
  void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
    assert (UseCompressedClassPointers, "should only be used for compressed headers");
  
+   assert(CompressedKlassPointers::shift() != 0, "not lilliput?");
+ 
    switch (klass_decode_mode()) {
    case KlassDecodeZero:
!     if (dst != src) mov(dst, src);
      break;
  
    case KlassDecodeXor:
!     lsl(dst, src, LogKlassAlignmentInBytes);
!     eor(dst, dst, (uint64_t)CompressedKlassPointers::base());
      break;
  
    case KlassDecodeMovk: {
      const uint64_t shifted_base =
        (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
  
+     // Invalid base should have been gracefully handled via klass_decode_mode() in VM initialization.
+     assert((shifted_base & 0xffff0000ffffffff) == 0, "incompatible base");
+ 
      if (dst != src) movw(dst, src);
      movk(dst, shifted_base >> 32, 32);
!     lsl(dst, dst, LogKlassAlignmentInBytes);
      break;
    }
  
    case KlassDecodeNone:
      ShouldNotReachHere();

*** 6147,5 ***
--- 6185,60 ---
        fmovd(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
      else
        strd(src.first()->as_FloatRegister(), Address(sp, reg2offset_out(dst.first())));
    }
  }
+ 
+ // Attempt to fast-lock an object. Fall-through on success, branch to slow label
+ // on failure.
+ // Registers:
+ //  - obj: the object to be locked
+ //  - hdr: the header, already loaded from obj, will be destroyed
+ //  - t1, t2, t3: temporary registers, will be destroyed
+ void MacroAssembler::fast_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow, bool rt_check_stack) {
+   assert(UseFastLocking, "only used with fast-locking");
+   assert_different_registers(obj, hdr, t1, t2);
+ 
+   if (rt_check_stack) {
+     // Check if we would have space on lock-stack for the object.
+     ldr(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
+     ldr(t2, Address(rthread, JavaThread::lock_stack_limit_offset()));
+     cmp(t1, t2);
+     br(Assembler::GE, slow);
+   }
+ 
+   // Load (object->mark() | 1) into hdr
+   orr(hdr, hdr, markWord::unlocked_value);
+   // Clear lock-bits, into t2
+   eor(t2, hdr, markWord::unlocked_value);
+   // Try to swing header from unlocked to locked
+   cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
+           /*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
+   br(Assembler::NE, slow);
+ 
+   // After successful lock, push object on lock-stack
+   ldr(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
+   str(obj, Address(t1, 0));
+   add(t1, t1, oopSize);
+   str(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
+ }
+ 
+ void MacroAssembler::fast_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
+   assert(UseFastLocking, "only used with fast-locking");
+   assert_different_registers(obj, hdr, t1, t2);
+ 
+   // Load the expected old header (lock-bits cleared to indicate 'locked') into hdr
+   andr(hdr, hdr, ~markWord::lock_mask_in_place);
+ 
+   // Load the new header (unlocked) into t1
+   orr(t1, hdr, markWord::unlocked_value);
+ 
+   // Try to swing header from locked to unlocked
+   cmpxchg(obj, hdr, t1, Assembler::xword,
+           /*acquire*/ true, /*release*/ true, /*weak*/ false, t2);
+   br(Assembler::NE, slow);
+ 
+   // After successful unlock, pop object from lock-stack
+   ldr(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
+   sub(t1, t1, oopSize);
+   str(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
+ }
< prev index next >