33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/cardTable.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/tlab_globals.hpp"
39 #include "interpreter/bytecodeHistogram.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "compiler/compileTask.hpp"
42 #include "compiler/disassembler.hpp"
43 #include "memory/resourceArea.hpp"
44 #include "memory/universe.hpp"
45 #include "nativeInst_aarch64.hpp"
46 #include "oops/accessDecorators.hpp"
47 #include "oops/compressedOops.inline.hpp"
48 #include "oops/klass.inline.hpp"
49 #include "runtime/biasedLocking.hpp"
50 #include "runtime/icache.hpp"
51 #include "runtime/interfaceSupport.inline.hpp"
52 #include "runtime/jniHandles.inline.hpp"
53 #include "runtime/sharedRuntime.hpp"
54 #include "runtime/stubRoutines.hpp"
55 #include "runtime/thread.hpp"
56 #include "utilities/powerOfTwo.hpp"
57 #ifdef COMPILER1
58 #include "c1/c1_LIRAssembler.hpp"
59 #endif
60 #ifdef COMPILER2
61 #include "oops/oop.hpp"
62 #include "opto/compile.hpp"
63 #include "opto/node.hpp"
64 #include "opto/output.hpp"
65 #endif
66
67 #ifdef PRODUCT
68 #define BLOCK_COMMENT(str) /* nothing */
69 #else
70 #define BLOCK_COMMENT(str) block_comment(str)
71 #endif
72 #define STOP(str) stop(str);
3788 adrp(rscratch1, src2, offset);
3789 ldr(rscratch1, Address(rscratch1, offset));
3790 cmp(src1, rscratch1);
3791 }
3792
3793 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3794 cmp(obj1, obj2);
3795 }
3796
3797 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3798 load_method_holder(rresult, rmethod);
3799 ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3800 }
3801
3802 void MacroAssembler::load_method_holder(Register holder, Register method) {
3803 ldr(holder, Address(method, Method::const_offset())); // ConstMethod*
3804 ldr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
3805 ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3806 }
3807
3808 void MacroAssembler::load_klass(Register dst, Register src) {
3809 if (UseCompressedClassPointers) {
3810 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3811 decode_klass_not_null(dst);
3812 } else {
3813 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3814 }
3815 }
3816
3817 // ((OopHandle)result).resolve();
3818 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3819 // OopHandle::resolve is an indirection.
3820 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3821 }
3822
3823 // ((WeakHandle)result).resolve();
3824 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3825 assert_different_registers(rresult, rtmp);
3826 Label resolved;
3827
3828 // A null weak handle resolves to null.
3829 cbz(rresult, resolved);
3830
3831 // Only 64 bit platforms support GCs that require a tmp register
3832 // Only IN_HEAP loads require a thread_tmp register
3833 // WeakHandle::resolve is an indirection like jweak.
3834 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3835 rresult, Address(rresult), rtmp, /*tmp_thread*/noreg);
3836 bind(resolved);
3837 }
3838
3839 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3840 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3841 ldr(dst, Address(rmethod, Method::const_offset()));
3842 ldr(dst, Address(dst, ConstMethod::constants_offset()));
3843 ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3844 ldr(dst, Address(dst, mirror_offset));
3845 resolve_oop_handle(dst, tmp);
3846 }
3847
3848 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3849 if (UseCompressedClassPointers) {
3850 ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3851 if (CompressedKlassPointers::base() == NULL) {
3852 cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3853 return;
3854 } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3855 && CompressedKlassPointers::shift() == 0) {
3856 // Only the bottom 32 bits matter
3857 cmpw(trial_klass, tmp);
3858 return;
3859 }
3860 decode_klass_not_null(tmp);
3861 } else {
3862 ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3863 }
3864 cmp(trial_klass, tmp);
3865 }
3866
3867 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3868 load_klass(dst, src);
3869 ldr(dst, Address(dst, Klass::prototype_header_offset()));
3870 }
3871
3872 void MacroAssembler::store_klass(Register dst, Register src) {
3873 // FIXME: Should this be a store release? concurrent gcs assumes
3874 // klass length is valid if klass field is not null.
3875 if (UseCompressedClassPointers) {
3876 encode_klass_not_null(src);
3877 strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3878 } else {
3879 str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3880 }
3881 }
3882
3883 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3884 if (UseCompressedClassPointers) {
3885 // Store to klass gap in destination
3886 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3887 }
3888 }
3889
3890 // Algorithm must match CompressedOops::encode.
3891 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3892 #ifdef ASSERT
3893 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3894 #endif
3895 verify_oop(s, "broken oop in encode_heap_oop");
3896 if (CompressedOops::base() == NULL) {
3897 if (CompressedOops::shift() != 0) {
3898 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3899 lsr(d, s, LogMinObjAlignmentInBytes);
3900 } else {
3901 mov(d, s);
3902 }
3903 } else {
3904 subs(d, s, rheapbase);
3905 csel(d, d, zr, Assembler::HS);
3906 lsr(d, d, LogMinObjAlignmentInBytes);
3907
3908 /* Old algorithm: is this any worse?
3909 Label nonnull;
5349 }
5350 #endif
5351
5352 void MacroAssembler::spin_wait() {
5353 for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) {
5354 switch (VM_Version::spin_wait_desc().inst()) {
5355 case SpinWait::NOP:
5356 nop();
5357 break;
5358 case SpinWait::ISB:
5359 isb();
5360 break;
5361 case SpinWait::YIELD:
5362 yield();
5363 break;
5364 default:
5365 ShouldNotReachHere();
5366 }
5367 }
5368 }
|
33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/cardTable.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/tlab_globals.hpp"
39 #include "interpreter/bytecodeHistogram.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "compiler/compileTask.hpp"
42 #include "compiler/disassembler.hpp"
43 #include "memory/resourceArea.hpp"
44 #include "memory/universe.hpp"
45 #include "nativeInst_aarch64.hpp"
46 #include "oops/accessDecorators.hpp"
47 #include "oops/compressedOops.inline.hpp"
48 #include "oops/klass.inline.hpp"
49 #include "runtime/biasedLocking.hpp"
50 #include "runtime/icache.hpp"
51 #include "runtime/interfaceSupport.inline.hpp"
52 #include "runtime/jniHandles.inline.hpp"
53 #include "runtime/objectMonitor.hpp"
54 #include "runtime/sharedRuntime.hpp"
55 #include "runtime/stubRoutines.hpp"
56 #include "runtime/thread.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #ifdef COMPILER1
59 #include "c1/c1_LIRAssembler.hpp"
60 #endif
61 #ifdef COMPILER2
62 #include "oops/oop.hpp"
63 #include "opto/compile.hpp"
64 #include "opto/node.hpp"
65 #include "opto/output.hpp"
66 #endif
67
68 #ifdef PRODUCT
69 #define BLOCK_COMMENT(str) /* nothing */
70 #else
71 #define BLOCK_COMMENT(str) block_comment(str)
72 #endif
73 #define STOP(str) stop(str);
3789 adrp(rscratch1, src2, offset);
3790 ldr(rscratch1, Address(rscratch1, offset));
3791 cmp(src1, rscratch1);
3792 }
3793
3794 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3795 cmp(obj1, obj2);
3796 }
3797
3798 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3799 load_method_holder(rresult, rmethod);
3800 ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3801 }
3802
3803 void MacroAssembler::load_method_holder(Register holder, Register method) {
3804 ldr(holder, Address(method, Method::const_offset())); // ConstMethod*
3805 ldr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
3806 ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3807 }
3808
3809 // Loads the obj's Klass* into dst.
3810 // src and dst must be distinct registers
3811 // Preserves all registers (incl src, rscratch1 and rscratch2), but clobbers condition flags
3812 void MacroAssembler::load_nklass(Register dst, Register src) {
3813 assert(UseCompressedClassPointers, "expects UseCompressedClassPointers");
3814
3815 if (!UseCompactObjectHeaders) {
3816 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3817 return;
3818 }
3819
3820 Label fast;
3821
3822 // Check if we can take the (common) fast path, if obj is unlocked.
3823 ldr(dst, Address(src, oopDesc::mark_offset_in_bytes()));
3824 tbz(dst, exact_log2(markWord::monitor_value), fast);
3825
3826 // Fetch displaced header
3827 ldr(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
3828
3829 // Fast-path: shift and decode Klass*.
3830 bind(fast);
3831 lsr(dst, dst, markWord::klass_shift);
3832 }
3833
3834 void MacroAssembler::load_klass(Register dst, Register src, bool null_check_src) {
3835 if (null_check_src) {
3836 if (UseCompactObjectHeaders) {
3837 null_check(src, oopDesc::mark_offset_in_bytes());
3838 } else {
3839 null_check(src, oopDesc::klass_offset_in_bytes());
3840 }
3841 }
3842
3843 if (UseCompressedClassPointers) {
3844 if (UseCompactObjectHeaders) {
3845 load_nklass(dst, src);
3846 } else {
3847 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3848 }
3849 decode_klass_not_null(dst);
3850 } else {
3851 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3852 }
3853 }
3854
3855 // ((OopHandle)result).resolve();
3856 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3857 // OopHandle::resolve is an indirection.
3858 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3859 }
3860
3861 // ((WeakHandle)result).resolve();
3862 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3863 assert_different_registers(rresult, rtmp);
3864 Label resolved;
3865
3866 // A null weak handle resolves to null.
3867 cbz(rresult, resolved);
3868
3869 // Only 64 bit platforms support GCs that require a tmp register
3870 // Only IN_HEAP loads require a thread_tmp register
3871 // WeakHandle::resolve is an indirection like jweak.
3872 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3873 rresult, Address(rresult), rtmp, /*tmp_thread*/noreg);
3874 bind(resolved);
3875 }
3876
3877 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3878 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3879 ldr(dst, Address(rmethod, Method::const_offset()));
3880 ldr(dst, Address(dst, ConstMethod::constants_offset()));
3881 ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3882 ldr(dst, Address(dst, mirror_offset));
3883 resolve_oop_handle(dst, tmp);
3884 }
3885
3886 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3887 assert_different_registers(oop, trial_klass, tmp);
3888 if (UseCompressedClassPointers) {
3889 if (UseCompactObjectHeaders) {
3890 load_nklass(tmp, oop);
3891 } else {
3892 ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3893 }
3894 if (CompressedKlassPointers::base() == NULL) {
3895 cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3896 return;
3897 } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3898 && CompressedKlassPointers::shift() == 0) {
3899 // Only the bottom 32 bits matter
3900 cmpw(trial_klass, tmp);
3901 return;
3902 }
3903 decode_klass_not_null(tmp);
3904 } else {
3905 ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3906 }
3907 cmp(trial_klass, tmp);
3908 }
3909
3910 void MacroAssembler::store_klass(Register dst, Register src) {
3911 // FIXME: Should this be a store release? concurrent gcs assumes
3912 // klass length is valid if klass field is not null.
3913 if (UseCompressedClassPointers) {
3914 encode_klass_not_null(src);
3915 strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3916 } else {
3917 str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3918 }
3919 }
3920
3921 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3922 if (UseCompressedClassPointers) {
3923 // Store to klass gap in destination
3924 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3925 }
3926 }
3927
3928 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3929 load_klass(dst, src);
3930 ldr(dst, Address(dst, Klass::prototype_header_offset()));
3931 }
3932
3933 // Algorithm must match CompressedOops::encode.
3934 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3935 #ifdef ASSERT
3936 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3937 #endif
3938 verify_oop(s, "broken oop in encode_heap_oop");
3939 if (CompressedOops::base() == NULL) {
3940 if (CompressedOops::shift() != 0) {
3941 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3942 lsr(d, s, LogMinObjAlignmentInBytes);
3943 } else {
3944 mov(d, s);
3945 }
3946 } else {
3947 subs(d, s, rheapbase);
3948 csel(d, d, zr, Assembler::HS);
3949 lsr(d, d, LogMinObjAlignmentInBytes);
3950
3951 /* Old algorithm: is this any worse?
3952 Label nonnull;
5392 }
5393 #endif
5394
5395 void MacroAssembler::spin_wait() {
5396 for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) {
5397 switch (VM_Version::spin_wait_desc().inst()) {
5398 case SpinWait::NOP:
5399 nop();
5400 break;
5401 case SpinWait::ISB:
5402 isb();
5403 break;
5404 case SpinWait::YIELD:
5405 yield();
5406 break;
5407 default:
5408 ShouldNotReachHere();
5409 }
5410 }
5411 }
5412
5413 // Attempt to fast-lock an object. Fall-through on success, branch to slow label
5414 // on failure.
5415 // Registers:
5416 // - obj: the object to be locked
5417 // - hdr: the header, already loaded from obj, will be destroyed
5418 // - t1, t2, t3: temporary registers, will be destroyed
5419 void MacroAssembler::fast_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow, bool rt_check_stack) {
5420 assert(UseFastLocking, "only used with fast-locking");
5421 assert_different_registers(obj, hdr, t1, t2);
5422
5423 if (rt_check_stack) {
5424 // Check if we would have space on lock-stack for the object.
5425 ldr(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
5426 ldr(t2, Address(rthread, JavaThread::lock_stack_limit_offset()));
5427 cmp(t1, t2);
5428 br(Assembler::GE, slow);
5429 }
5430
5431 // Load (object->mark() | 1) into hdr
5432 orr(hdr, hdr, markWord::unlocked_value);
5433 // Clear lock-bits, into t2
5434 eor(t2, hdr, markWord::unlocked_value);
5435 // Try to swing header from unlocked to locked
5436 cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
5437 /*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
5438 br(Assembler::NE, slow);
5439
5440 // After successful lock, push object on lock-stack
5441 ldr(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
5442 str(obj, Address(t1, 0));
5443 add(t1, t1, oopSize);
5444 str(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
5445 }
5446
5447 void MacroAssembler::fast_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
5448 assert(UseFastLocking, "only used with fast-locking");
5449 assert_different_registers(obj, hdr, t1, t2);
5450
5451 // Load the expected old header (lock-bits cleared to indicate 'locked') into hdr
5452 andr(hdr, hdr, ~markWord::lock_mask_in_place);
5453
5454 // Load the new header (unlocked) into t1
5455 orr(t1, hdr, markWord::unlocked_value);
5456
5457 // Try to swing header from locked to unlocked
5458 cmpxchg(obj, hdr, t1, Assembler::xword,
5459 /*acquire*/ true, /*release*/ true, /*weak*/ false, t2);
5460 br(Assembler::NE, slow);
5461
5462 // After successful unlock, pop object from lock-stack
5463 ldr(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
5464 sub(t1, t1, oopSize);
5465 str(t1, Address(rthread, JavaThread::lock_stack_current_offset()));
5466 }
|