1 /*
2 * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include <sys/types.h>
27
28 #include "precompiled.hpp"
29 #include "jvm.h"
30 #include "asm/assembler.hpp"
31 #include "asm/assembler.inline.hpp"
32 #include "ci/ciEnv.hpp"
33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/cardTable.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/tlab_globals.hpp"
39 #include "interpreter/bytecodeHistogram.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "compiler/compileTask.hpp"
42 #include "compiler/disassembler.hpp"
43 #include "memory/resourceArea.hpp"
44 #include "memory/universe.hpp"
45 #include "nativeInst_aarch64.hpp"
46 #include "oops/accessDecorators.hpp"
47 #include "oops/compressedOops.inline.hpp"
48 #include "oops/klass.inline.hpp"
49 #include "runtime/biasedLocking.hpp"
50 #include "runtime/icache.hpp"
51 #include "runtime/interfaceSupport.inline.hpp"
52 #include "runtime/jniHandles.inline.hpp"
53 #include "runtime/sharedRuntime.hpp"
54 #include "runtime/stubRoutines.hpp"
55 #include "runtime/thread.hpp"
56 #include "utilities/powerOfTwo.hpp"
57 #ifdef COMPILER1
58 #include "c1/c1_LIRAssembler.hpp"
59 #endif
60 #ifdef COMPILER2
61 #include "oops/oop.hpp"
62 #include "opto/compile.hpp"
63 #include "opto/node.hpp"
64 #include "opto/output.hpp"
65 #endif
66
67 #ifdef PRODUCT
68 #define BLOCK_COMMENT(str) /* nothing */
69 #else
70 #define BLOCK_COMMENT(str) block_comment(str)
71 #endif
72 #define STOP(str) stop(str);
73 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
74
75 // Patch any kind of instruction; there may be several instructions.
76 // Return the total length (in bytes) of the instructions.
77 int MacroAssembler::pd_patch_instruction_size(address branch, address target) {
78 int instructions = 1;
79 assert((uint64_t)target < (1ull << 48), "48-bit overflow in address constant");
80 intptr_t offset = (target - branch) >> 2;
81 unsigned insn = *(unsigned*)branch;
82 if ((Instruction_aarch64::extract(insn, 29, 24) & 0b111011) == 0b011000) {
83 // Load register (literal)
84 Instruction_aarch64::spatch(branch, 23, 5, offset);
85 } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {
86 // Unconditional branch (immediate)
2472 b(*fail);
2473 }
2474
2475 // A generic CAS; success or failure is in the EQ flag. A weak CAS
2476 // doesn't retry and may fail spuriously. If the oldval is wanted,
2477 // Pass a register for the result, otherwise pass noreg.
2478
2479 // Clobbers rscratch1
2480 void MacroAssembler::cmpxchg(Register addr, Register expected,
2481 Register new_val,
2482 enum operand_size size,
2483 bool acquire, bool release,
2484 bool weak,
2485 Register result) {
2486 if (result == noreg) result = rscratch1;
2487 BLOCK_COMMENT("cmpxchg {");
2488 if (UseLSE) {
2489 mov(result, expected);
2490 lse_cas(result, new_val, addr, size, acquire, release, /*not_pair*/ true);
2491 compare_eq(result, expected, size);
2492 } else {
2493 Label retry_load, done;
2494 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
2495 prfm(Address(addr), PSTL1STRM);
2496 bind(retry_load);
2497 load_exclusive(result, addr, size, acquire);
2498 compare_eq(result, expected, size);
2499 br(Assembler::NE, done);
2500 store_exclusive(rscratch1, new_val, addr, size, release);
2501 if (weak) {
2502 cmpw(rscratch1, 0u); // If the store fails, return NE to our caller.
2503 } else {
2504 cbnzw(rscratch1, retry_load);
2505 }
2506 bind(done);
2507 }
2508 BLOCK_COMMENT("} cmpxchg");
2509 }
2510
2511 // A generic comparison. Only compares for equality, clobbers rscratch1.
3788 adrp(rscratch1, src2, offset);
3789 ldr(rscratch1, Address(rscratch1, offset));
3790 cmp(src1, rscratch1);
3791 }
3792
3793 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3794 cmp(obj1, obj2);
3795 }
3796
3797 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3798 load_method_holder(rresult, rmethod);
3799 ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3800 }
3801
3802 void MacroAssembler::load_method_holder(Register holder, Register method) {
3803 ldr(holder, Address(method, Method::const_offset())); // ConstMethod*
3804 ldr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
3805 ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3806 }
3807
3808 void MacroAssembler::load_klass(Register dst, Register src) {
3809 if (UseCompressedClassPointers) {
3810 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3811 decode_klass_not_null(dst);
3812 } else {
3813 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3814 }
3815 }
3816
3817 // ((OopHandle)result).resolve();
3818 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3819 // OopHandle::resolve is an indirection.
3820 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3821 }
3822
3823 // ((WeakHandle)result).resolve();
3824 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3825 assert_different_registers(rresult, rtmp);
3826 Label resolved;
3827
3828 // A null weak handle resolves to null.
3829 cbz(rresult, resolved);
3830
3831 // Only 64 bit platforms support GCs that require a tmp register
3832 // Only IN_HEAP loads require a thread_tmp register
3833 // WeakHandle::resolve is an indirection like jweak.
3834 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3835 rresult, Address(rresult), rtmp, /*tmp_thread*/noreg);
3836 bind(resolved);
3837 }
3838
3839 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3840 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3841 ldr(dst, Address(rmethod, Method::const_offset()));
3842 ldr(dst, Address(dst, ConstMethod::constants_offset()));
3843 ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3844 ldr(dst, Address(dst, mirror_offset));
3845 resolve_oop_handle(dst, tmp);
3846 }
3847
3848 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3849 if (UseCompressedClassPointers) {
3850 ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3851 if (CompressedKlassPointers::base() == NULL) {
3852 cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3853 return;
3854 } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3855 && CompressedKlassPointers::shift() == 0) {
3856 // Only the bottom 32 bits matter
3857 cmpw(trial_klass, tmp);
3858 return;
3859 }
3860 decode_klass_not_null(tmp);
3861 } else {
3862 ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3863 }
3864 cmp(trial_klass, tmp);
3865 }
3866
3867 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3868 load_klass(dst, src);
3869 ldr(dst, Address(dst, Klass::prototype_header_offset()));
3870 }
3871
3872 void MacroAssembler::store_klass(Register dst, Register src) {
3873 // FIXME: Should this be a store release? concurrent gcs assumes
3874 // klass length is valid if klass field is not null.
3875 if (UseCompressedClassPointers) {
3876 encode_klass_not_null(src);
3877 strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3878 } else {
3879 str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3880 }
3881 }
3882
3883 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3884 if (UseCompressedClassPointers) {
3885 // Store to klass gap in destination
3886 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3887 }
3888 }
3889
3890 // Algorithm must match CompressedOops::encode.
3891 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3892 #ifdef ASSERT
3893 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3894 #endif
3895 verify_oop(s, "broken oop in encode_heap_oop");
3896 if (CompressedOops::base() == NULL) {
3897 if (CompressedOops::shift() != 0) {
3898 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3899 lsr(d, s, LogMinObjAlignmentInBytes);
3900 } else {
3901 mov(d, s);
3902 }
3903 } else {
3904 subs(d, s, rheapbase);
3905 csel(d, d, zr, Assembler::HS);
3906 lsr(d, d, LogMinObjAlignmentInBytes);
3907
3908 /* Old algorithm: is this any worse?
3909 Label nonnull;
5362 }
5363 #endif
5364
5365 void MacroAssembler::spin_wait() {
5366 for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) {
5367 switch (VM_Version::spin_wait_desc().inst()) {
5368 case SpinWait::NOP:
5369 nop();
5370 break;
5371 case SpinWait::ISB:
5372 isb();
5373 break;
5374 case SpinWait::YIELD:
5375 yield();
5376 break;
5377 default:
5378 ShouldNotReachHere();
5379 }
5380 }
5381 }
|
1 /*
2 * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "jvm.h"
28 #include "asm/assembler.hpp"
29 #include "asm/assembler.inline.hpp"
30 #include "ci/ciEnv.hpp"
31 #include "gc/shared/barrierSet.hpp"
32 #include "gc/shared/barrierSetAssembler.hpp"
33 #include "gc/shared/cardTableBarrierSet.hpp"
34 #include "gc/shared/cardTable.hpp"
35 #include "gc/shared/collectedHeap.hpp"
36 #include "gc/shared/tlab_globals.hpp"
37 #include "interpreter/bytecodeHistogram.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "compiler/compileTask.hpp"
40 #include "compiler/disassembler.hpp"
41 #include "memory/resourceArea.hpp"
42 #include "memory/universe.hpp"
43 #include "nativeInst_aarch64.hpp"
44 #include "oops/accessDecorators.hpp"
45 #include "oops/compressedOops.inline.hpp"
46 #include "oops/klass.inline.hpp"
47 #include "runtime/biasedLocking.hpp"
48 #include "runtime/icache.hpp"
49 #include "runtime/interfaceSupport.inline.hpp"
50 #include "runtime/jniHandles.inline.hpp"
51 #include "runtime/objectMonitor.hpp"
52 #include "runtime/sharedRuntime.hpp"
53 #include "runtime/stubRoutines.hpp"
54 #include "runtime/thread.hpp"
55 #include "utilities/globalDefinitions.hpp"
56 #include "utilities/powerOfTwo.hpp"
57 #ifdef COMPILER1
58 #include "c1/c1_LIRAssembler.hpp"
59 #endif
60 #ifdef COMPILER2
61 #include "oops/oop.hpp"
62 #include "opto/compile.hpp"
63 #include "opto/node.hpp"
64 #include "opto/output.hpp"
65 #endif
66
67 #include <sys/types.h>
68
69 #ifdef PRODUCT
70 #define BLOCK_COMMENT(str) /* nothing */
71 #else
72 #define BLOCK_COMMENT(str) block_comment(str)
73 #endif
74 #define STOP(str) stop(str);
75 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
76
77 // Patch any kind of instruction; there may be several instructions.
78 // Return the total length (in bytes) of the instructions.
79 int MacroAssembler::pd_patch_instruction_size(address branch, address target) {
80 int instructions = 1;
81 assert((uint64_t)target < (1ull << 48), "48-bit overflow in address constant");
82 intptr_t offset = (target - branch) >> 2;
83 unsigned insn = *(unsigned*)branch;
84 if ((Instruction_aarch64::extract(insn, 29, 24) & 0b111011) == 0b011000) {
85 // Load register (literal)
86 Instruction_aarch64::spatch(branch, 23, 5, offset);
87 } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {
88 // Unconditional branch (immediate)
2474 b(*fail);
2475 }
2476
2477 // A generic CAS; success or failure is in the EQ flag. A weak CAS
2478 // doesn't retry and may fail spuriously. If the oldval is wanted,
2479 // Pass a register for the result, otherwise pass noreg.
2480
2481 // Clobbers rscratch1
2482 void MacroAssembler::cmpxchg(Register addr, Register expected,
2483 Register new_val,
2484 enum operand_size size,
2485 bool acquire, bool release,
2486 bool weak,
2487 Register result) {
2488 if (result == noreg) result = rscratch1;
2489 BLOCK_COMMENT("cmpxchg {");
2490 if (UseLSE) {
2491 mov(result, expected);
2492 lse_cas(result, new_val, addr, size, acquire, release, /*not_pair*/ true);
2493 compare_eq(result, expected, size);
2494 #ifdef ASSERT
2495 // Poison rscratch1 which is written on !UseLSE branch
2496 mov(rscratch1, 0x1f1f1f1f1f1f1f1f);
2497 #endif
2498 } else {
2499 Label retry_load, done;
2500 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
2501 prfm(Address(addr), PSTL1STRM);
2502 bind(retry_load);
2503 load_exclusive(result, addr, size, acquire);
2504 compare_eq(result, expected, size);
2505 br(Assembler::NE, done);
2506 store_exclusive(rscratch1, new_val, addr, size, release);
2507 if (weak) {
2508 cmpw(rscratch1, 0u); // If the store fails, return NE to our caller.
2509 } else {
2510 cbnzw(rscratch1, retry_load);
2511 }
2512 bind(done);
2513 }
2514 BLOCK_COMMENT("} cmpxchg");
2515 }
2516
2517 // A generic comparison. Only compares for equality, clobbers rscratch1.
3794 adrp(rscratch1, src2, offset);
3795 ldr(rscratch1, Address(rscratch1, offset));
3796 cmp(src1, rscratch1);
3797 }
3798
3799 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3800 cmp(obj1, obj2);
3801 }
3802
3803 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3804 load_method_holder(rresult, rmethod);
3805 ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3806 }
3807
3808 void MacroAssembler::load_method_holder(Register holder, Register method) {
3809 ldr(holder, Address(method, Method::const_offset())); // ConstMethod*
3810 ldr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
3811 ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3812 }
3813
3814 // Loads the obj's Klass* into dst.
3815 // src and dst must be distinct registers
3816 // Preserves all registers (incl src, rscratch1 and rscratch2), but clobbers condition flags
3817 void MacroAssembler::load_nklass(Register dst, Register src) {
3818 assert(UseCompressedClassPointers, "expects UseCompressedClassPointers");
3819
3820 if (!UseCompactObjectHeaders) {
3821 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3822 return;
3823 }
3824
3825 Label fast;
3826
3827 // Check if we can take the (common) fast path, if obj is unlocked.
3828 ldr(dst, Address(src, oopDesc::mark_offset_in_bytes()));
3829 tbz(dst, exact_log2(markWord::monitor_value), fast);
3830
3831 // Fetch displaced header
3832 ldr(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
3833
3834 // Fast-path: shift and decode Klass*.
3835 bind(fast);
3836 lsr(dst, dst, markWord::klass_shift);
3837 }
3838
3839 void MacroAssembler::load_klass(Register dst, Register src, bool null_check_src) {
3840 if (null_check_src) {
3841 if (UseCompactObjectHeaders) {
3842 null_check(src, oopDesc::mark_offset_in_bytes());
3843 } else {
3844 null_check(src, oopDesc::klass_offset_in_bytes());
3845 }
3846 }
3847
3848 if (UseCompressedClassPointers) {
3849 if (UseCompactObjectHeaders) {
3850 load_nklass(dst, src);
3851 } else {
3852 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3853 }
3854 decode_klass_not_null(dst);
3855 } else {
3856 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3857 }
3858 }
3859
3860 // ((OopHandle)result).resolve();
3861 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3862 // OopHandle::resolve is an indirection.
3863 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3864 }
3865
3866 // ((WeakHandle)result).resolve();
3867 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3868 assert_different_registers(rresult, rtmp);
3869 Label resolved;
3870
3871 // A null weak handle resolves to null.
3872 cbz(rresult, resolved);
3873
3874 // Only 64 bit platforms support GCs that require a tmp register
3875 // Only IN_HEAP loads require a thread_tmp register
3876 // WeakHandle::resolve is an indirection like jweak.
3877 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3878 rresult, Address(rresult), rtmp, /*tmp_thread*/noreg);
3879 bind(resolved);
3880 }
3881
3882 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3883 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3884 ldr(dst, Address(rmethod, Method::const_offset()));
3885 ldr(dst, Address(dst, ConstMethod::constants_offset()));
3886 ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3887 ldr(dst, Address(dst, mirror_offset));
3888 resolve_oop_handle(dst, tmp);
3889 }
3890
3891 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3892 assert_different_registers(oop, trial_klass, tmp);
3893 if (UseCompressedClassPointers) {
3894 if (UseCompactObjectHeaders) {
3895 load_nklass(tmp, oop);
3896 } else {
3897 ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3898 }
3899 if (CompressedKlassPointers::base() == NULL) {
3900 cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3901 return;
3902 } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3903 && CompressedKlassPointers::shift() == 0) {
3904 // Only the bottom 32 bits matter
3905 cmpw(trial_klass, tmp);
3906 return;
3907 }
3908 decode_klass_not_null(tmp);
3909 } else {
3910 ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3911 }
3912 cmp(trial_klass, tmp);
3913 }
3914
3915 void MacroAssembler::store_klass(Register dst, Register src) {
3916 // FIXME: Should this be a store release? concurrent gcs assumes
3917 // klass length is valid if klass field is not null.
3918 if (UseCompressedClassPointers) {
3919 encode_klass_not_null(src);
3920 strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3921 } else {
3922 str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3923 }
3924 }
3925
3926 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3927 if (UseCompressedClassPointers) {
3928 // Store to klass gap in destination
3929 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3930 }
3931 }
3932
3933 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3934 load_klass(dst, src);
3935 ldr(dst, Address(dst, Klass::prototype_header_offset()));
3936 }
3937
3938 // Algorithm must match CompressedOops::encode.
3939 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3940 #ifdef ASSERT
3941 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3942 #endif
3943 verify_oop(s, "broken oop in encode_heap_oop");
3944 if (CompressedOops::base() == NULL) {
3945 if (CompressedOops::shift() != 0) {
3946 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3947 lsr(d, s, LogMinObjAlignmentInBytes);
3948 } else {
3949 mov(d, s);
3950 }
3951 } else {
3952 subs(d, s, rheapbase);
3953 csel(d, d, zr, Assembler::HS);
3954 lsr(d, d, LogMinObjAlignmentInBytes);
3955
3956 /* Old algorithm: is this any worse?
3957 Label nonnull;
5410 }
5411 #endif
5412
5413 void MacroAssembler::spin_wait() {
5414 for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) {
5415 switch (VM_Version::spin_wait_desc().inst()) {
5416 case SpinWait::NOP:
5417 nop();
5418 break;
5419 case SpinWait::ISB:
5420 isb();
5421 break;
5422 case SpinWait::YIELD:
5423 yield();
5424 break;
5425 default:
5426 ShouldNotReachHere();
5427 }
5428 }
5429 }
5430
5431 // Implements lightweight-locking.
5432 //
5433 // - obj: the object to be locked
5434 // - t1, t2, t3: temporary registers, will be destroyed
5435 // - slow: branched to if locking fails, absolute offset may larger than 32KB (imm14 encoding).
5436 void MacroAssembler::lightweight_lock(Register obj, Register t1, Register t2, Register t3, Label& slow) {
5437 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
5438 assert_different_registers(obj, t1, t2, t3, rscratch1);
5439
5440 Label push;
5441 const Register top = t1;
5442 const Register mark = t2;
5443 const Register t = t3;
5444
5445 // Preload the markWord. It is important that this is the first
5446 // instruction emitted as it is part of C1's null check semantics.
5447 ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
5448
5449 // Check if the lock-stack is full.
5450 ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
5451 cmpw(top, (unsigned)LockStack::end_offset());
5452 br(Assembler::GE, slow);
5453
5454 // Check for recursion.
5455 subw(t, top, oopSize);
5456 ldr(t, Address(rthread, t));
5457 cmp(obj, t);
5458 br(Assembler::EQ, push);
5459
5460 // Check header for monitor (0b10).
5461 tst(mark, markWord::monitor_value);
5462 br(Assembler::NE, slow);
5463
5464 // Try to lock. Transition lock bits 0b01 => 0b00
5465 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
5466 orr(mark, mark, markWord::unlocked_value);
5467 eor(t, mark, markWord::unlocked_value);
5468 cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::xword,
5469 /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
5470 br(Assembler::NE, slow);
5471
5472 bind(push);
5473 // After successful lock, push object on lock-stack.
5474 str(obj, Address(rthread, top));
5475 addw(top, top, oopSize);
5476 strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
5477 }
5478
5479 // Implements lightweight-unlocking.
5480 //
5481 // - obj: the object to be unlocked
5482 // - t1, t2, t3: temporary registers
5483 // - slow: branched to if unlocking fails, absolute offset may larger than 32KB (imm14 encoding).
5484 void MacroAssembler::lightweight_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow) {
5485 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
5486 // cmpxchg clobbers rscratch1.
5487 assert_different_registers(obj, t1, t2, t3, rscratch1);
5488
5489 #ifdef ASSERT
5490 {
5491 // Check for lock-stack underflow.
5492 Label stack_ok;
5493 ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
5494 cmpw(t1, (unsigned)LockStack::start_offset());
5495 br(Assembler::GE, stack_ok);
5496 STOP("Lock-stack underflow");
5497 bind(stack_ok);
5498 }
5499 #endif
5500
5501 Label unlocked, push_and_slow;
5502 const Register top = t1;
5503 const Register mark = t2;
5504 const Register t = t3;
5505
5506 // Check if obj is top of lock-stack.
5507 ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
5508 subw(top, top, oopSize);
5509 ldr(t, Address(rthread, top));
5510 cmp(obj, t);
5511 br(Assembler::NE, slow);
5512
5513 // Pop lock-stack.
5514 DEBUG_ONLY(str(zr, Address(rthread, top));)
5515 strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
5516
5517 // Check if recursive.
5518 subw(t, top, oopSize);
5519 ldr(t, Address(rthread, t));
5520 cmp(obj, t);
5521 br(Assembler::EQ, unlocked);
5522
5523 // Not recursive. Check header for monitor (0b10).
5524 ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
5525 tbnz(mark, log2i_exact(markWord::monitor_value), push_and_slow);
5526
5527 #ifdef ASSERT
5528 // Check header not unlocked (0b01).
5529 Label not_unlocked;
5530 tbz(mark, log2i_exact(markWord::unlocked_value), not_unlocked);
5531 stop("lightweight_unlock already unlocked");
5532 bind(not_unlocked);
5533 #endif
5534
5535 // Try to unlock. Transition lock bits 0b00 => 0b01
5536 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
5537 orr(t, mark, markWord::unlocked_value);
5538 cmpxchg(obj, mark, t, Assembler::xword,
5539 /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
5540 br(Assembler::EQ, unlocked);
5541
5542 bind(push_and_slow);
5543 // Restore lock-stack and handle the unlock in runtime.
5544 DEBUG_ONLY(str(obj, Address(rthread, top));)
5545 addw(top, top, oopSize);
5546 strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
5547 b(slow);
5548
5549 bind(unlocked);
5550 }
|