33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/cardTable.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/tlab_globals.hpp"
39 #include "interpreter/bytecodeHistogram.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "compiler/compileTask.hpp"
42 #include "compiler/disassembler.hpp"
43 #include "memory/resourceArea.hpp"
44 #include "memory/universe.hpp"
45 #include "nativeInst_aarch64.hpp"
46 #include "oops/accessDecorators.hpp"
47 #include "oops/compressedOops.inline.hpp"
48 #include "oops/klass.inline.hpp"
49 #include "runtime/biasedLocking.hpp"
50 #include "runtime/icache.hpp"
51 #include "runtime/interfaceSupport.inline.hpp"
52 #include "runtime/jniHandles.inline.hpp"
53 #include "runtime/sharedRuntime.hpp"
54 #include "runtime/stubRoutines.hpp"
55 #include "runtime/thread.hpp"
56 #include "utilities/powerOfTwo.hpp"
57 #ifdef COMPILER1
58 #include "c1/c1_LIRAssembler.hpp"
59 #endif
60 #ifdef COMPILER2
61 #include "oops/oop.hpp"
62 #include "opto/compile.hpp"
63 #include "opto/node.hpp"
64 #include "opto/output.hpp"
65 #endif
66
67 #ifdef PRODUCT
68 #define BLOCK_COMMENT(str) /* nothing */
69 #else
70 #define BLOCK_COMMENT(str) block_comment(str)
71 #endif
72 #define STOP(str) stop(str);
3788 adrp(rscratch1, src2, offset);
3789 ldr(rscratch1, Address(rscratch1, offset));
3790 cmp(src1, rscratch1);
3791 }
3792
3793 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3794 cmp(obj1, obj2);
3795 }
3796
3797 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3798 load_method_holder(rresult, rmethod);
3799 ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3800 }
3801
3802 void MacroAssembler::load_method_holder(Register holder, Register method) {
3803 ldr(holder, Address(method, Method::const_offset())); // ConstMethod*
3804 ldr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
3805 ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3806 }
3807
3808 void MacroAssembler::load_klass(Register dst, Register src) {
3809 if (UseCompressedClassPointers) {
3810 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3811 decode_klass_not_null(dst);
3812 } else {
3813 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3814 }
3815 }
3816
3817 // ((OopHandle)result).resolve();
3818 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3819 // OopHandle::resolve is an indirection.
3820 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3821 }
3822
3823 // ((WeakHandle)result).resolve();
3824 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3825 assert_different_registers(rresult, rtmp);
3826 Label resolved;
3827
3828 // A null weak handle resolves to null.
3829 cbz(rresult, resolved);
3830
3831 // Only 64 bit platforms support GCs that require a tmp register
3832 // Only IN_HEAP loads require a thread_tmp register
3833 // WeakHandle::resolve is an indirection like jweak.
3834 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3835 rresult, Address(rresult), rtmp, /*tmp_thread*/noreg);
3836 bind(resolved);
3837 }
3838
3839 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3840 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3841 ldr(dst, Address(rmethod, Method::const_offset()));
3842 ldr(dst, Address(dst, ConstMethod::constants_offset()));
3843 ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3844 ldr(dst, Address(dst, mirror_offset));
3845 resolve_oop_handle(dst, tmp);
3846 }
3847
3848 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3849 if (UseCompressedClassPointers) {
3850 ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3851 if (CompressedKlassPointers::base() == NULL) {
3852 cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3853 return;
3854 } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3855 && CompressedKlassPointers::shift() == 0) {
3856 // Only the bottom 32 bits matter
3857 cmpw(trial_klass, tmp);
3858 return;
3859 }
3860 decode_klass_not_null(tmp);
3861 } else {
3862 ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3863 }
3864 cmp(trial_klass, tmp);
3865 }
3866
3867 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3868 load_klass(dst, src);
3869 ldr(dst, Address(dst, Klass::prototype_header_offset()));
3870 }
3871
3872 void MacroAssembler::store_klass(Register dst, Register src) {
3873 // FIXME: Should this be a store release? concurrent gcs assumes
3874 // klass length is valid if klass field is not null.
3875 if (UseCompressedClassPointers) {
3876 encode_klass_not_null(src);
3877 strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3878 } else {
3879 str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3880 }
3881 }
3882
3883 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3884 if (UseCompressedClassPointers) {
3885 // Store to klass gap in destination
3886 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3887 }
3888 }
3889
3890 // Algorithm must match CompressedOops::encode.
3891 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3892 #ifdef ASSERT
3893 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3894 #endif
3895 verify_oop(s, "broken oop in encode_heap_oop");
3896 if (CompressedOops::base() == NULL) {
3897 if (CompressedOops::shift() != 0) {
3898 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3899 lsr(d, s, LogMinObjAlignmentInBytes);
3900 } else {
3901 mov(d, s);
3902 }
3903 } else {
3904 subs(d, s, rheapbase);
3905 csel(d, d, zr, Assembler::HS);
3906 lsr(d, d, LogMinObjAlignmentInBytes);
3907
3908 /* Old algorithm: is this any worse?
3909 Label nonnull;
5362 }
5363 #endif
5364
5365 void MacroAssembler::spin_wait() {
5366 for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) {
5367 switch (VM_Version::spin_wait_desc().inst()) {
5368 case SpinWait::NOP:
5369 nop();
5370 break;
5371 case SpinWait::ISB:
5372 isb();
5373 break;
5374 case SpinWait::YIELD:
5375 yield();
5376 break;
5377 default:
5378 ShouldNotReachHere();
5379 }
5380 }
5381 }
|
33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/cardTable.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/tlab_globals.hpp"
39 #include "interpreter/bytecodeHistogram.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "compiler/compileTask.hpp"
42 #include "compiler/disassembler.hpp"
43 #include "memory/resourceArea.hpp"
44 #include "memory/universe.hpp"
45 #include "nativeInst_aarch64.hpp"
46 #include "oops/accessDecorators.hpp"
47 #include "oops/compressedOops.inline.hpp"
48 #include "oops/klass.inline.hpp"
49 #include "runtime/biasedLocking.hpp"
50 #include "runtime/icache.hpp"
51 #include "runtime/interfaceSupport.inline.hpp"
52 #include "runtime/jniHandles.inline.hpp"
53 #include "runtime/objectMonitor.hpp"
54 #include "runtime/sharedRuntime.hpp"
55 #include "runtime/stubRoutines.hpp"
56 #include "runtime/thread.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #ifdef COMPILER1
59 #include "c1/c1_LIRAssembler.hpp"
60 #endif
61 #ifdef COMPILER2
62 #include "oops/oop.hpp"
63 #include "opto/compile.hpp"
64 #include "opto/node.hpp"
65 #include "opto/output.hpp"
66 #endif
67
68 #ifdef PRODUCT
69 #define BLOCK_COMMENT(str) /* nothing */
70 #else
71 #define BLOCK_COMMENT(str) block_comment(str)
72 #endif
73 #define STOP(str) stop(str);
3789 adrp(rscratch1, src2, offset);
3790 ldr(rscratch1, Address(rscratch1, offset));
3791 cmp(src1, rscratch1);
3792 }
3793
3794 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3795 cmp(obj1, obj2);
3796 }
3797
3798 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3799 load_method_holder(rresult, rmethod);
3800 ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3801 }
3802
3803 void MacroAssembler::load_method_holder(Register holder, Register method) {
3804 ldr(holder, Address(method, Method::const_offset())); // ConstMethod*
3805 ldr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
3806 ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3807 }
3808
3809 // Loads the obj's Klass* into dst.
3810 // src and dst must be distinct registers
3811 // Preserves all registers (incl src, rscratch1 and rscratch2), but clobbers condition flags
3812 void MacroAssembler::load_nklass(Register dst, Register src) {
3813 assert(UseCompressedClassPointers, "expects UseCompressedClassPointers");
3814
3815 if (!UseCompactObjectHeaders) {
3816 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3817 return;
3818 }
3819
3820 Label fast;
3821
3822 // Check if we can take the (common) fast path, if obj is unlocked.
3823 ldr(dst, Address(src, oopDesc::mark_offset_in_bytes()));
3824 tbz(dst, exact_log2(markWord::monitor_value), fast);
3825
3826 // Fetch displaced header
3827 ldr(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
3828
3829 // Fast-path: shift and decode Klass*.
3830 bind(fast);
3831 lsr(dst, dst, markWord::klass_shift);
3832 }
3833
3834 void MacroAssembler::load_klass(Register dst, Register src, bool null_check_src) {
3835 if (null_check_src) {
3836 if (UseCompactObjectHeaders) {
3837 null_check(src, oopDesc::mark_offset_in_bytes());
3838 } else {
3839 null_check(src, oopDesc::klass_offset_in_bytes());
3840 }
3841 }
3842
3843 if (UseCompressedClassPointers) {
3844 if (UseCompactObjectHeaders) {
3845 load_nklass(dst, src);
3846 } else {
3847 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3848 }
3849 decode_klass_not_null(dst);
3850 } else {
3851 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3852 }
3853 }
3854
3855 // ((OopHandle)result).resolve();
3856 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3857 // OopHandle::resolve is an indirection.
3858 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3859 }
3860
3861 // ((WeakHandle)result).resolve();
3862 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3863 assert_different_registers(rresult, rtmp);
3864 Label resolved;
3865
3866 // A null weak handle resolves to null.
3867 cbz(rresult, resolved);
3868
3869 // Only 64 bit platforms support GCs that require a tmp register
3870 // Only IN_HEAP loads require a thread_tmp register
3871 // WeakHandle::resolve is an indirection like jweak.
3872 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3873 rresult, Address(rresult), rtmp, /*tmp_thread*/noreg);
3874 bind(resolved);
3875 }
3876
3877 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3878 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3879 ldr(dst, Address(rmethod, Method::const_offset()));
3880 ldr(dst, Address(dst, ConstMethod::constants_offset()));
3881 ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3882 ldr(dst, Address(dst, mirror_offset));
3883 resolve_oop_handle(dst, tmp);
3884 }
3885
3886 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3887 assert_different_registers(oop, trial_klass, tmp);
3888 if (UseCompressedClassPointers) {
3889 if (UseCompactObjectHeaders) {
3890 load_nklass(tmp, oop);
3891 } else {
3892 ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3893 }
3894 if (CompressedKlassPointers::base() == NULL) {
3895 cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3896 return;
3897 } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3898 && CompressedKlassPointers::shift() == 0) {
3899 // Only the bottom 32 bits matter
3900 cmpw(trial_klass, tmp);
3901 return;
3902 }
3903 decode_klass_not_null(tmp);
3904 } else {
3905 ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3906 }
3907 cmp(trial_klass, tmp);
3908 }
3909
3910 void MacroAssembler::store_klass(Register dst, Register src) {
3911 // FIXME: Should this be a store release? concurrent gcs assumes
3912 // klass length is valid if klass field is not null.
3913 if (UseCompressedClassPointers) {
3914 encode_klass_not_null(src);
3915 strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3916 } else {
3917 str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3918 }
3919 }
3920
3921 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3922 if (UseCompressedClassPointers) {
3923 // Store to klass gap in destination
3924 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3925 }
3926 }
3927
3928 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3929 load_klass(dst, src);
3930 ldr(dst, Address(dst, Klass::prototype_header_offset()));
3931 }
3932
3933 // Algorithm must match CompressedOops::encode.
3934 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3935 #ifdef ASSERT
3936 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3937 #endif
3938 verify_oop(s, "broken oop in encode_heap_oop");
3939 if (CompressedOops::base() == NULL) {
3940 if (CompressedOops::shift() != 0) {
3941 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3942 lsr(d, s, LogMinObjAlignmentInBytes);
3943 } else {
3944 mov(d, s);
3945 }
3946 } else {
3947 subs(d, s, rheapbase);
3948 csel(d, d, zr, Assembler::HS);
3949 lsr(d, d, LogMinObjAlignmentInBytes);
3950
3951 /* Old algorithm: is this any worse?
3952 Label nonnull;
5405 }
5406 #endif
5407
5408 void MacroAssembler::spin_wait() {
5409 for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) {
5410 switch (VM_Version::spin_wait_desc().inst()) {
5411 case SpinWait::NOP:
5412 nop();
5413 break;
5414 case SpinWait::ISB:
5415 isb();
5416 break;
5417 case SpinWait::YIELD:
5418 yield();
5419 break;
5420 default:
5421 ShouldNotReachHere();
5422 }
5423 }
5424 }
5425
5426 // Implements fast-locking.
5427 // Branches to slow upon failure to lock the object, with ZF cleared.
5428 // Falls through upon success with ZF set.
5429 //
5430 // - obj: the object to be locked
5431 // - hdr: the header, already loaded from obj, will be destroyed
5432 // - t1, t2: temporary registers, will be destroyed
5433 void MacroAssembler::fast_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
5434 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
5435 assert_different_registers(obj, hdr, t1, t2);
5436
5437 // Check if we would have space on lock-stack for the object.
5438 ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5439 cmpw(t1, (unsigned)LockStack::end_offset() - 1);
5440 br(Assembler::GT, slow);
5441
5442 // Load (object->mark() | 1) into hdr
5443 orr(hdr, hdr, markWord::unlocked_value);
5444 // Clear lock-bits, into t2
5445 eor(t2, hdr, markWord::unlocked_value);
5446 // Try to swing header from unlocked to locked
5447 cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
5448 /*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
5449 br(Assembler::NE, slow);
5450
5451 // After successful lock, push object on lock-stack
5452 ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5453 str(obj, Address(rthread, t1));
5454 addw(t1, t1, oopSize);
5455 strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5456 }
5457
5458 // Implements fast-unlocking.
5459 // Branches to slow upon failure, with ZF cleared.
5460 // Falls through upon success, with ZF set.
5461 //
5462 // - obj: the object to be unlocked
5463 // - hdr: the (pre-loaded) header of the object
5464 // - t1, t2: temporary registers
5465 void MacroAssembler::fast_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
5466 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
5467 assert_different_registers(obj, hdr, t1, t2);
5468
5469 #ifdef ASSERT
5470 {
5471 // The following checks rely on the fact that LockStack is only ever modified by
5472 // its owning thread, even if the lock got inflated concurrently; removal of LockStack
5473 // entries after inflation will happen delayed in that case.
5474
5475 // Check for lock-stack underflow.
5476 Label stack_ok;
5477 ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5478 cmpw(t1, (unsigned)LockStack::start_offset());
5479 br(Assembler::GT, stack_ok);
5480 STOP("Lock-stack underflow");
5481 bind(stack_ok);
5482 }
5483 {
5484 // Check if the top of the lock-stack matches the unlocked object.
5485 Label tos_ok;
5486 subw(t1, t1, oopSize);
5487 ldr(t1, Address(rthread, t1));
5488 cmpoop(t1, obj);
5489 br(Assembler::EQ, tos_ok);
5490 STOP("Top of lock-stack does not match the unlocked object");
5491 bind(tos_ok);
5492 }
5493 {
5494 // Check that hdr is fast-locked.
5495 Label hdr_ok;
5496 tst(hdr, markWord::lock_mask_in_place);
5497 br(Assembler::EQ, hdr_ok);
5498 STOP("Header is not fast-locked");
5499 bind(hdr_ok);
5500 }
5501 #endif
5502
5503 // Load the new header (unlocked) into t1
5504 orr(t1, hdr, markWord::unlocked_value);
5505
5506 // Try to swing header from locked to unlocked
5507 cmpxchg(obj, hdr, t1, Assembler::xword,
5508 /*acquire*/ true, /*release*/ true, /*weak*/ false, t2);
5509 br(Assembler::NE, slow);
5510
5511 // After successful unlock, pop object from lock-stack
5512 ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5513 subw(t1, t1, oopSize);
5514 #ifdef ASSERT
5515 str(zr, Address(rthread, t1));
5516 #endif
5517 strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5518 }
|