31 #include "gc/shared/barrierSetAssembler.hpp"
32 #include "gc/shared/collectedHeap.hpp"
33 #include "gc/shared/tlab_globals.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "oops/arrayOop.hpp"
36 #include "oops/markWord.hpp"
37 #include "runtime/basicLock.hpp"
38 #include "runtime/globals.hpp"
39 #include "runtime/os.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #include "runtime/stubRoutines.hpp"
42 #include "utilities/checkedCast.hpp"
43 #include "utilities/globalDefinitions.hpp"
44
45 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case) {
46 const int aligned_mask = BytesPerWord -1;
47 const int hdr_offset = oopDesc::mark_offset_in_bytes();
48 assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction");
49 assert_different_registers(hdr, obj, disp_hdr, tmp);
50 int null_check_offset = -1;
51
52 verify_oop(obj);
53
54 // save object being locked into the BasicObjectLock
55 movptr(Address(disp_hdr, BasicObjectLock::obj_offset()), obj);
56
57 null_check_offset = offset();
58
59 if (DiagnoseSyncOnValueBasedClasses != 0) {
60 load_klass(hdr, obj, rscratch1);
61 movl(hdr, Address(hdr, Klass::access_flags_offset()));
62 testl(hdr, JVM_ACC_IS_VALUE_BASED_CLASS);
63 jcc(Assembler::notZero, slow_case);
64 }
65
66 if (LockingMode == LM_LIGHTWEIGHT) {
67 #ifdef _LP64
68 const Register thread = r15_thread;
69 #else
70 const Register thread = disp_hdr;
71 get_thread(thread);
72 #endif
73 lightweight_lock(obj, hdr, thread, tmp, slow_case);
74 } else if (LockingMode == LM_LEGACY) {
75 Label done;
76 // Load object header
77 movptr(hdr, Address(obj, hdr_offset));
78 // and mark it as unlocked
79 orptr(hdr, markWord::unlocked_value);
80 // save unlocked object header into the displaced header location on the stack
81 movptr(Address(disp_hdr, 0), hdr);
82 // test if object header is still the same (i.e. unlocked), and if so, store the
83 // displaced header address in the object header - if it is not the same, get the
84 // object header instead
85 MacroAssembler::lock(); // must be immediately before cmpxchg!
86 cmpxchgptr(disp_hdr, Address(obj, hdr_offset));
87 // if the object header was the same, we're done
88 jcc(Assembler::equal, done);
89 // if the object header was not the same, it is now in the hdr register
90 // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
91 //
92 // 1) (hdr & aligned_mask) == 0
93 // 2) rsp <= hdr
94 // 3) hdr <= rsp + page_size
95 //
96 // these 3 tests can be done by evaluating the following expression:
97 //
98 // (hdr - rsp) & (aligned_mask - page_size)
99 //
100 // assuming both the stack pointer and page_size have their least
101 // significant 2 bits cleared and page_size is a power of 2
102 subptr(hdr, rsp);
103 andptr(hdr, aligned_mask - (int)os::vm_page_size());
104 // for recursive locking, the result is zero => save it in the displaced header
105 // location (null in the displaced hdr location indicates recursive locking)
106 movptr(Address(disp_hdr, 0), hdr);
107 // otherwise we don't care about the result and handle locking via runtime call
108 jcc(Assembler::notZero, slow_case);
109 // done
110 bind(done);
111 }
112
113 inc_held_monitor_count();
114
115 return null_check_offset;
116 }
117
118 void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
119 const int aligned_mask = BytesPerWord -1;
120 const int hdr_offset = oopDesc::mark_offset_in_bytes();
121 assert(disp_hdr == rax, "disp_hdr must be rax, for the cmpxchg instruction");
122 assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
123 Label done;
124
125 if (LockingMode != LM_LIGHTWEIGHT) {
126 // load displaced header
127 movptr(hdr, Address(disp_hdr, 0));
128 // if the loaded hdr is null we had recursive locking
129 testptr(hdr, hdr);
130 // if we had recursive locking, we are done
131 jcc(Assembler::zero, done);
132 }
133
136 verify_oop(obj);
137
138 if (LockingMode == LM_LIGHTWEIGHT) {
139 #ifdef _LP64
140 lightweight_unlock(obj, disp_hdr, r15_thread, hdr, slow_case);
141 #else
142 // This relies on the implementation of lightweight_unlock being able to handle
143 // that the reg_rax and thread Register parameters may alias each other.
144 get_thread(disp_hdr);
145 lightweight_unlock(obj, disp_hdr, disp_hdr, hdr, slow_case);
146 #endif
147 } else if (LockingMode == LM_LEGACY) {
148 // test if object header is pointing to the displaced header, and if so, restore
149 // the displaced header in the object - if the object header is not pointing to
150 // the displaced header, get the object header instead
151 MacroAssembler::lock(); // must be immediately before cmpxchg!
152 cmpxchgptr(hdr, Address(obj, hdr_offset));
153 // if the object header was not pointing to the displaced header,
154 // we do unlocking via runtime call
155 jcc(Assembler::notEqual, slow_case);
156 // done
157 }
158 bind(done);
159 dec_held_monitor_count();
160 }
161
162
163 // Defines obj, preserves var_size_in_bytes
164 void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
165 if (UseTLAB) {
166 tlab_allocate(noreg, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
167 } else {
168 jmp(slow_case);
169 }
170 }
171
172
173 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
174 assert_different_registers(obj, klass, len);
175 movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
176 #ifdef _LP64
177 if (UseCompressedClassPointers) { // Take care not to kill klass
178 movptr(t1, klass);
179 encode_klass_not_null(t1, rscratch1);
|
31 #include "gc/shared/barrierSetAssembler.hpp"
32 #include "gc/shared/collectedHeap.hpp"
33 #include "gc/shared/tlab_globals.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "oops/arrayOop.hpp"
36 #include "oops/markWord.hpp"
37 #include "runtime/basicLock.hpp"
38 #include "runtime/globals.hpp"
39 #include "runtime/os.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #include "runtime/stubRoutines.hpp"
42 #include "utilities/checkedCast.hpp"
43 #include "utilities/globalDefinitions.hpp"
44
45 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case) {
46 const int aligned_mask = BytesPerWord -1;
47 const int hdr_offset = oopDesc::mark_offset_in_bytes();
48 assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction");
49 assert_different_registers(hdr, obj, disp_hdr, tmp);
50 int null_check_offset = -1;
51 Label count_locking, done;
52
53 verify_oop(obj);
54
55 // save object being locked into the BasicObjectLock
56 movptr(Address(disp_hdr, BasicObjectLock::obj_offset()), obj);
57
58 null_check_offset = offset();
59
60 if (DiagnoseSyncOnValueBasedClasses != 0) {
61 load_klass(hdr, obj, rscratch1);
62 movl(hdr, Address(hdr, Klass::access_flags_offset()));
63 testl(hdr, JVM_ACC_IS_VALUE_BASED_CLASS);
64 jcc(Assembler::notZero, slow_case);
65 }
66
67 if (LockingMode == LM_LIGHTWEIGHT) {
68 #ifdef _LP64
69 const Register thread = r15_thread;
70 #else
71 const Register thread = disp_hdr;
72 get_thread(thread);
73 #endif
74 lightweight_lock(obj, hdr, thread, tmp, slow_case);
75 } else if (LockingMode == LM_LEGACY) {
76 // Load object header
77 movptr(hdr, Address(obj, hdr_offset));
78 // and mark it as unlocked
79 orptr(hdr, markWord::unlocked_value);
80 // save unlocked object header into the displaced header location on the stack
81 movptr(Address(disp_hdr, 0), hdr);
82 // test if object header is still the same (i.e. unlocked), and if so, store the
83 // displaced header address in the object header - if it is not the same, get the
84 // object header instead
85 MacroAssembler::lock(); // must be immediately before cmpxchg!
86 cmpxchgptr(disp_hdr, Address(obj, hdr_offset));
87 // if the object header was the same, we're done
88 jcc(Assembler::equal, count_locking);
89 // if the object header was not the same, it is now in the hdr register
90 // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
91 //
92 // 1) (hdr & aligned_mask) == 0
93 // 2) rsp <= hdr
94 // 3) hdr <= rsp + page_size
95 //
96 // these 3 tests can be done by evaluating the following expression:
97 //
98 // (hdr - rsp) & (aligned_mask - page_size)
99 //
100 // assuming both the stack pointer and page_size have their least
101 // significant 2 bits cleared and page_size is a power of 2
102 subptr(hdr, rsp);
103 andptr(hdr, aligned_mask - (int)os::vm_page_size());
104 // for recursive locking, the result is zero => save it in the displaced header
105 // location (null in the displaced hdr location indicates recursive locking)
106 movptr(Address(disp_hdr, 0), hdr);
107 // otherwise we don't care about the result and handle locking via runtime call
108 jcc(Assembler::notZero, slow_case);
109 // done
110 }
111 jmp(done);
112
113 bind(count_locking);
114 inc_held_monitor_count();
115 bind(done);
116
117 return null_check_offset;
118 }
119
120 void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
121 const int aligned_mask = BytesPerWord -1;
122 const int hdr_offset = oopDesc::mark_offset_in_bytes();
123 assert(disp_hdr == rax, "disp_hdr must be rax, for the cmpxchg instruction");
124 assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
125 Label done;
126
127 if (LockingMode != LM_LIGHTWEIGHT) {
128 // load displaced header
129 movptr(hdr, Address(disp_hdr, 0));
130 // if the loaded hdr is null we had recursive locking
131 testptr(hdr, hdr);
132 // if we had recursive locking, we are done
133 jcc(Assembler::zero, done);
134 }
135
138 verify_oop(obj);
139
140 if (LockingMode == LM_LIGHTWEIGHT) {
141 #ifdef _LP64
142 lightweight_unlock(obj, disp_hdr, r15_thread, hdr, slow_case);
143 #else
144 // This relies on the implementation of lightweight_unlock being able to handle
145 // that the reg_rax and thread Register parameters may alias each other.
146 get_thread(disp_hdr);
147 lightweight_unlock(obj, disp_hdr, disp_hdr, hdr, slow_case);
148 #endif
149 } else if (LockingMode == LM_LEGACY) {
150 // test if object header is pointing to the displaced header, and if so, restore
151 // the displaced header in the object - if the object header is not pointing to
152 // the displaced header, get the object header instead
153 MacroAssembler::lock(); // must be immediately before cmpxchg!
154 cmpxchgptr(hdr, Address(obj, hdr_offset));
155 // if the object header was not pointing to the displaced header,
156 // we do unlocking via runtime call
157 jcc(Assembler::notEqual, slow_case);
158 dec_held_monitor_count();
159 // done
160 }
161 bind(done);
162 }
163
164
165 // Defines obj, preserves var_size_in_bytes
166 void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
167 if (UseTLAB) {
168 tlab_allocate(noreg, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
169 } else {
170 jmp(slow_case);
171 }
172 }
173
174
175 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
176 assert_different_registers(obj, klass, len);
177 movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
178 #ifdef _LP64
179 if (UseCompressedClassPointers) { // Take care not to kill klass
180 movptr(t1, klass);
181 encode_klass_not_null(t1, rscratch1);
|