< prev index next >

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

Print this page

  33 #include "gc/shared/barrierSet.hpp"
  34 #include "gc/shared/barrierSetAssembler.hpp"
  35 #include "gc/shared/cardTableBarrierSet.hpp"
  36 #include "gc/shared/cardTable.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "gc/shared/tlab_globals.hpp"
  39 #include "interpreter/bytecodeHistogram.hpp"
  40 #include "interpreter/interpreter.hpp"
  41 #include "compiler/compileTask.hpp"
  42 #include "compiler/disassembler.hpp"
  43 #include "memory/resourceArea.hpp"
  44 #include "memory/universe.hpp"
  45 #include "nativeInst_aarch64.hpp"
  46 #include "oops/accessDecorators.hpp"
  47 #include "oops/compressedOops.inline.hpp"
  48 #include "oops/klass.inline.hpp"
  49 #include "runtime/biasedLocking.hpp"
  50 #include "runtime/icache.hpp"
  51 #include "runtime/interfaceSupport.inline.hpp"
  52 #include "runtime/jniHandles.inline.hpp"

  53 #include "runtime/sharedRuntime.hpp"
  54 #include "runtime/stubRoutines.hpp"
  55 #include "runtime/thread.hpp"
  56 #include "utilities/powerOfTwo.hpp"
  57 #ifdef COMPILER1
  58 #include "c1/c1_LIRAssembler.hpp"
  59 #endif
  60 #ifdef COMPILER2
  61 #include "oops/oop.hpp"
  62 #include "opto/compile.hpp"
  63 #include "opto/node.hpp"
  64 #include "opto/output.hpp"
  65 #endif
  66 
  67 #ifdef PRODUCT
  68 #define BLOCK_COMMENT(str) /* nothing */
  69 #else
  70 #define BLOCK_COMMENT(str) block_comment(str)
  71 #endif
  72 #define STOP(str) stop(str);

3788   adrp(rscratch1, src2, offset);
3789   ldr(rscratch1, Address(rscratch1, offset));
3790   cmp(src1, rscratch1);
3791 }
3792 
3793 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3794   cmp(obj1, obj2);
3795 }
3796 
3797 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3798   load_method_holder(rresult, rmethod);
3799   ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3800 }
3801 
3802 void MacroAssembler::load_method_holder(Register holder, Register method) {
3803   ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
3804   ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
3805   ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3806 }
3807 
3808 void MacroAssembler::load_klass(Register dst, Register src) {
3809   if (UseCompressedClassPointers) {





3810     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
































3811     decode_klass_not_null(dst);
3812   } else {
3813     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3814   }
3815 }
3816 
3817 // ((OopHandle)result).resolve();
3818 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3819   // OopHandle::resolve is an indirection.
3820   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3821 }
3822 
3823 // ((WeakHandle)result).resolve();
3824 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3825   assert_different_registers(rresult, rtmp);
3826   Label resolved;
3827 
3828   // A null weak handle resolves to null.
3829   cbz(rresult, resolved);
3830 
3831   // Only 64 bit platforms support GCs that require a tmp register
3832   // Only IN_HEAP loads require a thread_tmp register
3833   // WeakHandle::resolve is an indirection like jweak.
3834   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3835                  rresult, Address(rresult), rtmp, /*tmp_thread*/noreg);
3836   bind(resolved);
3837 }
3838 
3839 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3840   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3841   ldr(dst, Address(rmethod, Method::const_offset()));
3842   ldr(dst, Address(dst, ConstMethod::constants_offset()));
3843   ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3844   ldr(dst, Address(dst, mirror_offset));
3845   resolve_oop_handle(dst, tmp);
3846 }
3847 
3848 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {

3849   if (UseCompressedClassPointers) {
3850     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));




3851     if (CompressedKlassPointers::base() == NULL) {
3852       cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3853       return;
3854     } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3855                && CompressedKlassPointers::shift() == 0) {
3856       // Only the bottom 32 bits matter
3857       cmpw(trial_klass, tmp);
3858       return;
3859     }
3860     decode_klass_not_null(tmp);
3861   } else {
3862     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3863   }
3864   cmp(trial_klass, tmp);
3865 }
3866 
3867 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3868   load_klass(dst, src);
3869   ldr(dst, Address(dst, Klass::prototype_header_offset()));
3870 }
3871 
3872 void MacroAssembler::store_klass(Register dst, Register src) {
3873   // FIXME: Should this be a store release?  concurrent gcs assumes
3874   // klass length is valid if klass field is not null.
3875   if (UseCompressedClassPointers) {
3876     encode_klass_not_null(src);
3877     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3878   } else {
3879     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3880   }
3881 }
3882 
3883 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3884   if (UseCompressedClassPointers) {
3885     // Store to klass gap in destination
3886     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3887   }
3888 }
3889 





3890 // Algorithm must match CompressedOops::encode.
3891 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3892 #ifdef ASSERT
3893   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3894 #endif
3895   verify_oop(s, "broken oop in encode_heap_oop");
3896   if (CompressedOops::base() == NULL) {
3897     if (CompressedOops::shift() != 0) {
3898       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3899       lsr(d, s, LogMinObjAlignmentInBytes);
3900     } else {
3901       mov(d, s);
3902     }
3903   } else {
3904     subs(d, s, rheapbase);
3905     csel(d, d, zr, Assembler::HS);
3906     lsr(d, d, LogMinObjAlignmentInBytes);
3907 
3908     /*  Old algorithm: is this any worse?
3909     Label nonnull;

5349 }
5350 #endif
5351 
5352 void MacroAssembler::spin_wait() {
5353   for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) {
5354     switch (VM_Version::spin_wait_desc().inst()) {
5355       case SpinWait::NOP:
5356         nop();
5357         break;
5358       case SpinWait::ISB:
5359         isb();
5360         break;
5361       case SpinWait::YIELD:
5362         yield();
5363         break;
5364       default:
5365         ShouldNotReachHere();
5366     }
5367   }
5368 }
























































  33 #include "gc/shared/barrierSet.hpp"
  34 #include "gc/shared/barrierSetAssembler.hpp"
  35 #include "gc/shared/cardTableBarrierSet.hpp"
  36 #include "gc/shared/cardTable.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "gc/shared/tlab_globals.hpp"
  39 #include "interpreter/bytecodeHistogram.hpp"
  40 #include "interpreter/interpreter.hpp"
  41 #include "compiler/compileTask.hpp"
  42 #include "compiler/disassembler.hpp"
  43 #include "memory/resourceArea.hpp"
  44 #include "memory/universe.hpp"
  45 #include "nativeInst_aarch64.hpp"
  46 #include "oops/accessDecorators.hpp"
  47 #include "oops/compressedOops.inline.hpp"
  48 #include "oops/klass.inline.hpp"
  49 #include "runtime/biasedLocking.hpp"
  50 #include "runtime/icache.hpp"
  51 #include "runtime/interfaceSupport.inline.hpp"
  52 #include "runtime/jniHandles.inline.hpp"
  53 #include "runtime/objectMonitor.hpp"
  54 #include "runtime/sharedRuntime.hpp"
  55 #include "runtime/stubRoutines.hpp"
  56 #include "runtime/thread.hpp"
  57 #include "utilities/powerOfTwo.hpp"
  58 #ifdef COMPILER1
  59 #include "c1/c1_LIRAssembler.hpp"
  60 #endif
  61 #ifdef COMPILER2
  62 #include "oops/oop.hpp"
  63 #include "opto/compile.hpp"
  64 #include "opto/node.hpp"
  65 #include "opto/output.hpp"
  66 #endif
  67 
  68 #ifdef PRODUCT
  69 #define BLOCK_COMMENT(str) /* nothing */
  70 #else
  71 #define BLOCK_COMMENT(str) block_comment(str)
  72 #endif
  73 #define STOP(str) stop(str);

3789   adrp(rscratch1, src2, offset);
3790   ldr(rscratch1, Address(rscratch1, offset));
3791   cmp(src1, rscratch1);
3792 }
3793 
3794 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3795   cmp(obj1, obj2);
3796 }
3797 
3798 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3799   load_method_holder(rresult, rmethod);
3800   ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3801 }
3802 
3803 void MacroAssembler::load_method_holder(Register holder, Register method) {
3804   ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
3805   ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
3806   ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3807 }
3808 
3809 // Loads the obj's Klass* into dst.
3810 // src and dst must be distinct registers
3811 // Preserves all registers (incl src, rscratch1 and rscratch2), but clobbers condition flags
3812 void MacroAssembler::load_nklass(Register dst, Register src) {
3813   assert(UseCompressedClassPointers, "expects UseCompressedClassPointers");
3814 
3815   if (!UseCompactObjectHeaders) {
3816     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3817     return;
3818   }
3819 
3820   Label fast;
3821 
3822   // Check if we can take the (common) fast path, if obj is unlocked.
3823   ldr(dst, Address(src, oopDesc::mark_offset_in_bytes()));
3824   tbz(dst, exact_log2(markWord::monitor_value), fast);
3825 
3826   // Fetch displaced header
3827   ldr(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
3828 
3829   // Fast-path: shift and decode Klass*.
3830   bind(fast);
3831   lsr(dst, dst, markWord::klass_shift);
3832 }
3833 
3834 void MacroAssembler::load_klass(Register dst, Register src, bool null_check_src) {
3835   if (null_check_src) {
3836     if (UseCompactObjectHeaders) {
3837       null_check(src, oopDesc::mark_offset_in_bytes());
3838     } else {
3839       null_check(src, oopDesc::klass_offset_in_bytes());
3840     }
3841   }
3842 
3843   if (UseCompressedClassPointers) {
3844     if (UseCompactObjectHeaders) {
3845       load_nklass(dst, src);
3846     } else {
3847       ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3848     }
3849     decode_klass_not_null(dst);
3850   } else {
3851     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3852   }
3853 }
3854 
3855 // ((OopHandle)result).resolve();
3856 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3857   // OopHandle::resolve is an indirection.
3858   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3859 }
3860 
3861 // ((WeakHandle)result).resolve();
3862 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3863   assert_different_registers(rresult, rtmp);
3864   Label resolved;
3865 
3866   // A null weak handle resolves to null.
3867   cbz(rresult, resolved);
3868 
3869   // Only 64 bit platforms support GCs that require a tmp register
3870   // Only IN_HEAP loads require a thread_tmp register
3871   // WeakHandle::resolve is an indirection like jweak.
3872   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3873                  rresult, Address(rresult), rtmp, /*tmp_thread*/noreg);
3874   bind(resolved);
3875 }
3876 
3877 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3878   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3879   ldr(dst, Address(rmethod, Method::const_offset()));
3880   ldr(dst, Address(dst, ConstMethod::constants_offset()));
3881   ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3882   ldr(dst, Address(dst, mirror_offset));
3883   resolve_oop_handle(dst, tmp);
3884 }
3885 
3886 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3887   assert_different_registers(oop, trial_klass, tmp);
3888   if (UseCompressedClassPointers) {
3889     if (UseCompactObjectHeaders) {
3890       load_nklass(tmp, oop);
3891     } else {
3892       ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3893     }
3894     if (CompressedKlassPointers::base() == NULL) {
3895       cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3896       return;
3897     } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3898                && CompressedKlassPointers::shift() == 0) {
3899       // Only the bottom 32 bits matter
3900       cmpw(trial_klass, tmp);
3901       return;
3902     }
3903     decode_klass_not_null(tmp);
3904   } else {
3905     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3906   }
3907   cmp(trial_klass, tmp);
3908 }
3909 





3910 void MacroAssembler::store_klass(Register dst, Register src) {
3911   // FIXME: Should this be a store release?  concurrent gcs assumes
3912   // klass length is valid if klass field is not null.
3913   if (UseCompressedClassPointers) {
3914     encode_klass_not_null(src);
3915     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3916   } else {
3917     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3918   }
3919 }
3920 
3921 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3922   if (UseCompressedClassPointers) {
3923     // Store to klass gap in destination
3924     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3925   }
3926 }
3927 
3928 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3929   load_klass(dst, src);
3930   ldr(dst, Address(dst, Klass::prototype_header_offset()));
3931 }
3932 
3933 // Algorithm must match CompressedOops::encode.
3934 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3935 #ifdef ASSERT
3936   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3937 #endif
3938   verify_oop(s, "broken oop in encode_heap_oop");
3939   if (CompressedOops::base() == NULL) {
3940     if (CompressedOops::shift() != 0) {
3941       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3942       lsr(d, s, LogMinObjAlignmentInBytes);
3943     } else {
3944       mov(d, s);
3945     }
3946   } else {
3947     subs(d, s, rheapbase);
3948     csel(d, d, zr, Assembler::HS);
3949     lsr(d, d, LogMinObjAlignmentInBytes);
3950 
3951     /*  Old algorithm: is this any worse?
3952     Label nonnull;

5392 }
5393 #endif
5394 
5395 void MacroAssembler::spin_wait() {
5396   for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) {
5397     switch (VM_Version::spin_wait_desc().inst()) {
5398       case SpinWait::NOP:
5399         nop();
5400         break;
5401       case SpinWait::ISB:
5402         isb();
5403         break;
5404       case SpinWait::YIELD:
5405         yield();
5406         break;
5407       default:
5408         ShouldNotReachHere();
5409     }
5410   }
5411 }
5412 
5413 // Attempt to fast-lock an object. Fall-through on success, branch to slow label
5414 // on failure.
5415 // Registers:
5416 //  - obj: the object to be locked
5417 //  - hdr: the header, already loaded from obj, will be destroyed
5418 //  - t1, t2, t3: temporary registers, will be destroyed
5419 void MacroAssembler::fast_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow, bool rt_check_stack) {
5420   assert(UseFastLocking, "only used with fast-locking");
5421   assert_different_registers(obj, hdr, t1, t2);
5422 
5423   if (rt_check_stack) {
5424     // Check if we would have space on lock-stack for the object.
5425     ldr(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
5426     ldr(t2, Address(rthread, JavaThread::lock_stack_limit_offset()));
5427     cmp(t1, t2);
5428     br(Assembler::GE, slow);
5429   }
5430 
5431   // Load (object->mark() | 1) into hdr
5432   orr(hdr, hdr, markWord::unlocked_value);
5433   // Clear lock-bits, into t2
5434   eor(t2, hdr, markWord::unlocked_value);
5435   // Try to swing header from unlocked to locked
5436   cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
5437           /*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
5438   br(Assembler::NE, slow);
5439 
5440   // After successful lock, push object on lock-stack
5441   ldr(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
5442   str(obj, Address(t1, 0));
5443   add(t1, t1, oopSize);
5444   str(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
5445 }
5446 
5447 void MacroAssembler::fast_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
5448   assert(UseFastLocking, "only used with fast-locking");
5449   assert_different_registers(obj, hdr, t1, t2);
5450 
5451   // Load the expected old header (lock-bits cleared to indicate 'locked') into hdr
5452   andr(hdr, hdr, ~markWord::lock_mask_in_place);
5453 
5454   // Load the new header (unlocked) into t1
5455   orr(t1, hdr, markWord::unlocked_value);
5456 
5457   // Try to swing header from locked to unlocked
5458   cmpxchg(obj, hdr, t1, Assembler::xword,
5459           /*acquire*/ true, /*release*/ true, /*weak*/ false, t2);
5460   br(Assembler::NE, slow);
5461 
5462   // After successful unlock, pop object from lock-stack
5463   ldr(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
5464   sub(t1, t1, oopSize);
5465   str(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
5466 }
< prev index next >