< prev index next >

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

Print this page

  33 #include "gc/shared/barrierSet.hpp"
  34 #include "gc/shared/barrierSetAssembler.hpp"
  35 #include "gc/shared/cardTableBarrierSet.hpp"
  36 #include "gc/shared/cardTable.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "gc/shared/tlab_globals.hpp"
  39 #include "interpreter/bytecodeHistogram.hpp"
  40 #include "interpreter/interpreter.hpp"
  41 #include "compiler/compileTask.hpp"
  42 #include "compiler/disassembler.hpp"
  43 #include "memory/resourceArea.hpp"
  44 #include "memory/universe.hpp"
  45 #include "nativeInst_aarch64.hpp"
  46 #include "oops/accessDecorators.hpp"
  47 #include "oops/compressedOops.inline.hpp"
  48 #include "oops/klass.inline.hpp"
  49 #include "runtime/biasedLocking.hpp"
  50 #include "runtime/icache.hpp"
  51 #include "runtime/interfaceSupport.inline.hpp"
  52 #include "runtime/jniHandles.inline.hpp"

  53 #include "runtime/sharedRuntime.hpp"
  54 #include "runtime/stubRoutines.hpp"
  55 #include "runtime/thread.hpp"
  56 #include "utilities/powerOfTwo.hpp"
  57 #ifdef COMPILER1
  58 #include "c1/c1_LIRAssembler.hpp"
  59 #endif
  60 #ifdef COMPILER2
  61 #include "oops/oop.hpp"
  62 #include "opto/compile.hpp"
  63 #include "opto/node.hpp"
  64 #include "opto/output.hpp"
  65 #endif
  66 
  67 #ifdef PRODUCT
  68 #define BLOCK_COMMENT(str) /* nothing */
  69 #else
  70 #define BLOCK_COMMENT(str) block_comment(str)
  71 #endif
  72 #define STOP(str) stop(str);

3788   adrp(rscratch1, src2, offset);
3789   ldr(rscratch1, Address(rscratch1, offset));
3790   cmp(src1, rscratch1);
3791 }
3792 
3793 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3794   cmp(obj1, obj2);
3795 }
3796 
3797 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3798   load_method_holder(rresult, rmethod);
3799   ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3800 }
3801 
3802 void MacroAssembler::load_method_holder(Register holder, Register method) {
3803   ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
3804   ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
3805   ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3806 }
3807 
3808 void MacroAssembler::load_klass(Register dst, Register src) {
3809   if (UseCompressedClassPointers) {





3810     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
































3811     decode_klass_not_null(dst);
3812   } else {
3813     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3814   }
3815 }
3816 
3817 // ((OopHandle)result).resolve();
3818 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3819   // OopHandle::resolve is an indirection.
3820   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3821 }
3822 
3823 // ((WeakHandle)result).resolve();
3824 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3825   assert_different_registers(rresult, rtmp);
3826   Label resolved;
3827 
3828   // A null weak handle resolves to null.
3829   cbz(rresult, resolved);
3830 
3831   // Only 64 bit platforms support GCs that require a tmp register
3832   // Only IN_HEAP loads require a thread_tmp register
3833   // WeakHandle::resolve is an indirection like jweak.
3834   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3835                  rresult, Address(rresult), rtmp, /*tmp_thread*/noreg);
3836   bind(resolved);
3837 }
3838 
3839 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3840   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3841   ldr(dst, Address(rmethod, Method::const_offset()));
3842   ldr(dst, Address(dst, ConstMethod::constants_offset()));
3843   ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3844   ldr(dst, Address(dst, mirror_offset));
3845   resolve_oop_handle(dst, tmp);
3846 }
3847 
3848 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {

3849   if (UseCompressedClassPointers) {
3850     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));




3851     if (CompressedKlassPointers::base() == NULL) {
3852       cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3853       return;
3854     } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3855                && CompressedKlassPointers::shift() == 0) {
3856       // Only the bottom 32 bits matter
3857       cmpw(trial_klass, tmp);
3858       return;
3859     }
3860     decode_klass_not_null(tmp);
3861   } else {
3862     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3863   }
3864   cmp(trial_klass, tmp);
3865 }
3866 
3867 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3868   load_klass(dst, src);
3869   ldr(dst, Address(dst, Klass::prototype_header_offset()));
3870 }
3871 
3872 void MacroAssembler::store_klass(Register dst, Register src) {
3873   // FIXME: Should this be a store release?  concurrent gcs assumes
3874   // klass length is valid if klass field is not null.
3875   if (UseCompressedClassPointers) {
3876     encode_klass_not_null(src);
3877     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3878   } else {
3879     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3880   }
3881 }
3882 
3883 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3884   if (UseCompressedClassPointers) {
3885     // Store to klass gap in destination
3886     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3887   }
3888 }
3889 





3890 // Algorithm must match CompressedOops::encode.
3891 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3892 #ifdef ASSERT
3893   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3894 #endif
3895   verify_oop(s, "broken oop in encode_heap_oop");
3896   if (CompressedOops::base() == NULL) {
3897     if (CompressedOops::shift() != 0) {
3898       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3899       lsr(d, s, LogMinObjAlignmentInBytes);
3900     } else {
3901       mov(d, s);
3902     }
3903   } else {
3904     subs(d, s, rheapbase);
3905     csel(d, d, zr, Assembler::HS);
3906     lsr(d, d, LogMinObjAlignmentInBytes);
3907 
3908     /*  Old algorithm: is this any worse?
3909     Label nonnull;

5362 }
5363 #endif
5364 
5365 void MacroAssembler::spin_wait() {
5366   for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) {
5367     switch (VM_Version::spin_wait_desc().inst()) {
5368       case SpinWait::NOP:
5369         nop();
5370         break;
5371       case SpinWait::ISB:
5372         isb();
5373         break;
5374       case SpinWait::YIELD:
5375         yield();
5376         break;
5377       default:
5378         ShouldNotReachHere();
5379     }
5380   }
5381 }































































































  33 #include "gc/shared/barrierSet.hpp"
  34 #include "gc/shared/barrierSetAssembler.hpp"
  35 #include "gc/shared/cardTableBarrierSet.hpp"
  36 #include "gc/shared/cardTable.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "gc/shared/tlab_globals.hpp"
  39 #include "interpreter/bytecodeHistogram.hpp"
  40 #include "interpreter/interpreter.hpp"
  41 #include "compiler/compileTask.hpp"
  42 #include "compiler/disassembler.hpp"
  43 #include "memory/resourceArea.hpp"
  44 #include "memory/universe.hpp"
  45 #include "nativeInst_aarch64.hpp"
  46 #include "oops/accessDecorators.hpp"
  47 #include "oops/compressedOops.inline.hpp"
  48 #include "oops/klass.inline.hpp"
  49 #include "runtime/biasedLocking.hpp"
  50 #include "runtime/icache.hpp"
  51 #include "runtime/interfaceSupport.inline.hpp"
  52 #include "runtime/jniHandles.inline.hpp"
  53 #include "runtime/objectMonitor.hpp"
  54 #include "runtime/sharedRuntime.hpp"
  55 #include "runtime/stubRoutines.hpp"
  56 #include "runtime/thread.hpp"
  57 #include "utilities/powerOfTwo.hpp"
  58 #ifdef COMPILER1
  59 #include "c1/c1_LIRAssembler.hpp"
  60 #endif
  61 #ifdef COMPILER2
  62 #include "oops/oop.hpp"
  63 #include "opto/compile.hpp"
  64 #include "opto/node.hpp"
  65 #include "opto/output.hpp"
  66 #endif
  67 
  68 #ifdef PRODUCT
  69 #define BLOCK_COMMENT(str) /* nothing */
  70 #else
  71 #define BLOCK_COMMENT(str) block_comment(str)
  72 #endif
  73 #define STOP(str) stop(str);

3789   adrp(rscratch1, src2, offset);
3790   ldr(rscratch1, Address(rscratch1, offset));
3791   cmp(src1, rscratch1);
3792 }
3793 
3794 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3795   cmp(obj1, obj2);
3796 }
3797 
3798 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3799   load_method_holder(rresult, rmethod);
3800   ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3801 }
3802 
3803 void MacroAssembler::load_method_holder(Register holder, Register method) {
3804   ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
3805   ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
3806   ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3807 }
3808 
3809 // Loads the obj's Klass* into dst.
3810 // src and dst must be distinct registers
3811 // Preserves all registers (incl src, rscratch1 and rscratch2), but clobbers condition flags
3812 void MacroAssembler::load_nklass(Register dst, Register src) {
3813   assert(UseCompressedClassPointers, "expects UseCompressedClassPointers");
3814 
3815   if (!UseCompactObjectHeaders) {
3816     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3817     return;
3818   }
3819 
3820   Label fast;
3821 
3822   // Check if we can take the (common) fast path, if obj is unlocked.
3823   ldr(dst, Address(src, oopDesc::mark_offset_in_bytes()));
3824   tbz(dst, exact_log2(markWord::monitor_value), fast);
3825 
3826   // Fetch displaced header
3827   ldr(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
3828 
3829   // Fast-path: shift and decode Klass*.
3830   bind(fast);
3831   lsr(dst, dst, markWord::klass_shift);
3832 }
3833 
3834 void MacroAssembler::load_klass(Register dst, Register src, bool null_check_src) {
3835   if (null_check_src) {
3836     if (UseCompactObjectHeaders) {
3837       null_check(src, oopDesc::mark_offset_in_bytes());
3838     } else {
3839       null_check(src, oopDesc::klass_offset_in_bytes());
3840     }
3841   }
3842 
3843   if (UseCompressedClassPointers) {
3844     if (UseCompactObjectHeaders) {
3845       load_nklass(dst, src);
3846     } else {
3847       ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3848     }
3849     decode_klass_not_null(dst);
3850   } else {
3851     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3852   }
3853 }
3854 
3855 // ((OopHandle)result).resolve();
3856 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3857   // OopHandle::resolve is an indirection.
3858   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3859 }
3860 
3861 // ((WeakHandle)result).resolve();
3862 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3863   assert_different_registers(rresult, rtmp);
3864   Label resolved;
3865 
3866   // A null weak handle resolves to null.
3867   cbz(rresult, resolved);
3868 
3869   // Only 64 bit platforms support GCs that require a tmp register
3870   // Only IN_HEAP loads require a thread_tmp register
3871   // WeakHandle::resolve is an indirection like jweak.
3872   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3873                  rresult, Address(rresult), rtmp, /*tmp_thread*/noreg);
3874   bind(resolved);
3875 }
3876 
3877 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3878   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3879   ldr(dst, Address(rmethod, Method::const_offset()));
3880   ldr(dst, Address(dst, ConstMethod::constants_offset()));
3881   ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3882   ldr(dst, Address(dst, mirror_offset));
3883   resolve_oop_handle(dst, tmp);
3884 }
3885 
3886 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3887   assert_different_registers(oop, trial_klass, tmp);
3888   if (UseCompressedClassPointers) {
3889     if (UseCompactObjectHeaders) {
3890       load_nklass(tmp, oop);
3891     } else {
3892       ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3893     }
3894     if (CompressedKlassPointers::base() == NULL) {
3895       cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3896       return;
3897     } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3898                && CompressedKlassPointers::shift() == 0) {
3899       // Only the bottom 32 bits matter
3900       cmpw(trial_klass, tmp);
3901       return;
3902     }
3903     decode_klass_not_null(tmp);
3904   } else {
3905     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3906   }
3907   cmp(trial_klass, tmp);
3908 }
3909 





3910 void MacroAssembler::store_klass(Register dst, Register src) {
3911   // FIXME: Should this be a store release?  concurrent gcs assumes
3912   // klass length is valid if klass field is not null.
3913   if (UseCompressedClassPointers) {
3914     encode_klass_not_null(src);
3915     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3916   } else {
3917     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3918   }
3919 }
3920 
3921 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3922   if (UseCompressedClassPointers) {
3923     // Store to klass gap in destination
3924     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3925   }
3926 }
3927 
3928 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3929   load_klass(dst, src);
3930   ldr(dst, Address(dst, Klass::prototype_header_offset()));
3931 }
3932 
3933 // Algorithm must match CompressedOops::encode.
3934 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3935 #ifdef ASSERT
3936   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3937 #endif
3938   verify_oop(s, "broken oop in encode_heap_oop");
3939   if (CompressedOops::base() == NULL) {
3940     if (CompressedOops::shift() != 0) {
3941       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3942       lsr(d, s, LogMinObjAlignmentInBytes);
3943     } else {
3944       mov(d, s);
3945     }
3946   } else {
3947     subs(d, s, rheapbase);
3948     csel(d, d, zr, Assembler::HS);
3949     lsr(d, d, LogMinObjAlignmentInBytes);
3950 
3951     /*  Old algorithm: is this any worse?
3952     Label nonnull;

5405 }
5406 #endif
5407 
5408 void MacroAssembler::spin_wait() {
5409   for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) {
5410     switch (VM_Version::spin_wait_desc().inst()) {
5411       case SpinWait::NOP:
5412         nop();
5413         break;
5414       case SpinWait::ISB:
5415         isb();
5416         break;
5417       case SpinWait::YIELD:
5418         yield();
5419         break;
5420       default:
5421         ShouldNotReachHere();
5422     }
5423   }
5424 }
5425 
5426 // Implements fast-locking.
5427 // Branches to slow upon failure to lock the object, with ZF cleared.
5428 // Falls through upon success with ZF set.
5429 //
5430 //  - obj: the object to be locked
5431 //  - hdr: the header, already loaded from obj, will be destroyed
5432 //  - t1, t2: temporary registers, will be destroyed
5433 void MacroAssembler::fast_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
5434   assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
5435   assert_different_registers(obj, hdr, t1, t2);
5436 
5437   // Check if we would have space on lock-stack for the object.
5438   ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5439   cmpw(t1, (unsigned)LockStack::end_offset() - 1);
5440   br(Assembler::GT, slow);
5441 
5442   // Load (object->mark() | 1) into hdr
5443   orr(hdr, hdr, markWord::unlocked_value);
5444   // Clear lock-bits, into t2
5445   eor(t2, hdr, markWord::unlocked_value);
5446   // Try to swing header from unlocked to locked
5447   cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
5448           /*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
5449   br(Assembler::NE, slow);
5450 
5451   // After successful lock, push object on lock-stack
5452   ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5453   str(obj, Address(rthread, t1));
5454   addw(t1, t1, oopSize);
5455   strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5456 }
5457 
5458 // Implements fast-unlocking.
5459 // Branches to slow upon failure, with ZF cleared.
5460 // Falls through upon success, with ZF set.
5461 //
5462 // - obj: the object to be unlocked
5463 // - hdr: the (pre-loaded) header of the object
5464 // - t1, t2: temporary registers
5465 void MacroAssembler::fast_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
5466   assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
5467   assert_different_registers(obj, hdr, t1, t2);
5468 
5469 #ifdef ASSERT
5470   {
5471     // The following checks rely on the fact that LockStack is only ever modified by
5472     // its owning thread, even if the lock got inflated concurrently; removal of LockStack
5473     // entries after inflation will happen delayed in that case.
5474 
5475     // Check for lock-stack underflow.
5476     Label stack_ok;
5477     ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5478     cmpw(t1, (unsigned)LockStack::start_offset());
5479     br(Assembler::GT, stack_ok);
5480     STOP("Lock-stack underflow");
5481     bind(stack_ok);
5482   }
5483   {
5484     // Check if the top of the lock-stack matches the unlocked object.
5485     Label tos_ok;
5486     subw(t1, t1, oopSize);
5487     ldr(t1, Address(rthread, t1));
5488     cmpoop(t1, obj);
5489     br(Assembler::EQ, tos_ok);
5490     STOP("Top of lock-stack does not match the unlocked object");
5491     bind(tos_ok);
5492   }
5493   {
5494     // Check that hdr is fast-locked.
5495     Label hdr_ok;
5496     tst(hdr, markWord::lock_mask_in_place);
5497     br(Assembler::EQ, hdr_ok);
5498     STOP("Header is not fast-locked");
5499     bind(hdr_ok);
5500   }
5501 #endif
5502 
5503   // Load the new header (unlocked) into t1
5504   orr(t1, hdr, markWord::unlocked_value);
5505 
5506   // Try to swing header from locked to unlocked
5507   cmpxchg(obj, hdr, t1, Assembler::xword,
5508           /*acquire*/ true, /*release*/ true, /*weak*/ false, t2);
5509   br(Assembler::NE, slow);
5510 
5511   // After successful unlock, pop object from lock-stack
5512   ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5513   subw(t1, t1, oopSize);
5514 #ifdef ASSERT
5515   str(zr, Address(rthread, t1));
5516 #endif
5517   strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5518 }
< prev index next >