15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "asm/assembler.inline.hpp"
29 #include "opto/c2_MacroAssembler.hpp"
30 #include "opto/compile.hpp"
31 #include "opto/intrinsicnode.hpp"
32 #include "opto/output.hpp"
33 #include "opto/subnode.hpp"
34 #include "runtime/stubRoutines.hpp"
35
36 #ifdef PRODUCT
37 #define BLOCK_COMMENT(str) /* nothing */
38 #define STOP(error) stop(error)
39 #else
40 #define BLOCK_COMMENT(str) block_comment(str)
41 #define STOP(error) block_comment(error); stop(error)
42 #endif
43
44 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
45
46 void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg,
47 Register tmp1Reg, Register tmp2Reg, Register tmp3Reg) {
48 // Use cr register to indicate the fast_lock result: zero for success; non-zero for failure.
49 Register flag = t1;
50 Register oop = objectReg;
51 Register box = boxReg;
52 Register disp_hdr = tmp1Reg;
53 Register tmp = tmp2Reg;
54 Label cont;
55 Label object_has_monitor;
56 Label count, no_count;
57
58 assert_different_registers(oop, box, tmp, disp_hdr, flag, tmp3Reg, t0);
59
60 // Load markWord from object into displaced_header.
61 ld(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
62
63 if (DiagnoseSyncOnValueBasedClasses != 0) {
64 load_klass(flag, oop);
65 lwu(flag, Address(flag, Klass::access_flags_offset()));
66 test_bit(flag, flag, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
67 bnez(flag, cont, true /* is_far */);
68 }
69
70 // Check for existing monitor
71 test_bit(t0, disp_hdr, exact_log2(markWord::monitor_value));
72 bnez(t0, object_has_monitor);
73
74 if (LockingMode == LM_MONITOR) {
75 mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path
76 j(cont);
77 } else if (LockingMode == LM_LEGACY) {
78 // Set tmp to be (markWord of object | UNLOCK_VALUE).
79 ori(tmp, disp_hdr, markWord::unlocked_value);
80
81 // Initialize the box. (Must happen before we update the object mark!)
82 sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
83
84 // Compare object markWord with an unlocked value (tmp) and if
85 // equal exchange the stack address of our box with object markWord.
86 // On failure disp_hdr contains the possibly locked markWord.
87 cmpxchg(/*memory address*/oop, /*expected value*/tmp, /*new value*/box, Assembler::int64, Assembler::aq,
88 Assembler::rl, /*result*/disp_hdr);
89 mv(flag, zr);
90 beq(disp_hdr, tmp, cont); // prepare zero flag and goto cont if we won the cas
91
92 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
93
94 // If the compare-and-exchange succeeded, then we found an unlocked
95 // object, will have now locked it will continue at label cont
96 // We did not see an unlocked object so try the fast recursive case.
97
98 // Check if the owner is self by comparing the value in the
99 // markWord of object (disp_hdr) with the stack pointer.
100 sub(disp_hdr, disp_hdr, sp);
101 mv(tmp, (intptr_t) (~(os::vm_page_size()-1) | (uintptr_t)markWord::lock_mask_in_place));
102 // If (mark & lock_mask) == 0 and mark - sp < page_size, we are stack-locking and goto cont,
103 // hence we can store 0 as the displaced header in the box, which indicates that it is a
104 // recursive lock.
105 andr(tmp/*==0?*/, disp_hdr, tmp);
106 sd(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
107 mv(flag, tmp); // we can use the value of tmp as the result here
108 j(cont);
109 } else {
110 assert(LockingMode == LM_LIGHTWEIGHT, "");
111 Label slow;
112 lightweight_lock(oop, disp_hdr, tmp, tmp3Reg, slow);
113
114 // Indicate success on completion.
115 mv(flag, zr);
116 j(count);
117 bind(slow);
118 mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path
119 j(no_count);
120 }
121
122 // Handle existing monitor.
123 bind(object_has_monitor);
124 // The object's monitor m is unlocked iff m->owner == NULL,
125 // otherwise m->owner may contain a thread or a stack address.
126 //
127 // Try to CAS m->owner from NULL to current thread.
128 add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value));
129 cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/xthread, Assembler::int64, Assembler::aq,
130 Assembler::rl, /*result*/flag); // cas succeeds if flag == zr(expected)
131
132 if (LockingMode != LM_LIGHTWEIGHT) {
133 // Store a non-null value into the box to avoid looking like a re-entrant
134 // lock. The fast-path monitor unlock code checks for
135 // markWord::monitor_value so use markWord::unused_mark which has the
136 // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
137 mv(tmp, (address)markWord::unused_mark().value());
138 sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
139 }
140
141 beqz(flag, cont); // CAS success means locking succeeded
142
143 bne(flag, xthread, cont); // Check for recursive locking
144
145 // Recursive lock case
146 mv(flag, zr);
147 increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1, t0, tmp);
148
149 bind(cont);
150 // zero flag indicates success
151 // non-zero flag indicates failure
152 bnez(flag, no_count);
153
154 bind(count);
155 increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, t0, tmp);
156
157 bind(no_count);
158 }
159
160 void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg,
161 Register tmp1Reg, Register tmp2Reg) {
162 // Use cr register to indicate the fast_unlock result: zero for success; non-zero for failure.
163 Register flag = t1;
164 Register oop = objectReg;
165 Register box = boxReg;
166 Register disp_hdr = tmp1Reg;
167 Register tmp = tmp2Reg;
168 Label cont;
169 Label object_has_monitor;
170 Label count, no_count;
171
172 assert_different_registers(oop, box, tmp, disp_hdr, flag, t0);
173
174 if (LockingMode == LM_LEGACY) {
175 // Find the lock address and load the displaced header from the stack.
176 ld(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
177
178 // If the displaced header is 0, we have a recursive unlock.
179 mv(flag, disp_hdr);
180 beqz(disp_hdr, cont);
181 }
182
183 // Handle existing monitor.
184 ld(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
185 test_bit(t0, tmp, exact_log2(markWord::monitor_value));
186 bnez(t0, object_has_monitor);
187
188 if (LockingMode == LM_MONITOR) {
189 mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path
190 j(cont);
191 } else if (LockingMode == LM_LEGACY) {
192 // Check if it is still a light weight lock, this is true if we
193 // see the stack address of the basicLock in the markWord of the
194 // object.
195
196 cmpxchg(/*memory address*/oop, /*expected value*/box, /*new value*/disp_hdr, Assembler::int64, Assembler::relaxed,
197 Assembler::rl, /*result*/tmp);
198 xorr(flag, box, tmp); // box == tmp if cas succeeds
199 j(cont);
200 } else {
201 assert(LockingMode == LM_LIGHTWEIGHT, "");
202 Label slow;
203 lightweight_unlock(oop, tmp, box, disp_hdr, slow);
204
205 // Indicate success on completion.
206 mv(flag, zr);
207 j(count);
208 bind(slow);
209 mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path
210 j(no_count);
211 }
212
213 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
214
215 // Handle existing monitor.
216 bind(object_has_monitor);
217 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
218 add(tmp, tmp, -(int)markWord::monitor_value); // monitor
219
220 if (LockingMode == LM_LIGHTWEIGHT) {
221 // If the owner is anonymous, we need to fix it -- in an outline stub.
222 Register tmp2 = disp_hdr;
223 ld(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
224 test_bit(t0, tmp2, exact_log2(ObjectMonitor::ANONYMOUS_OWNER));
225 C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
226 Compile::current()->output()->add_stub(stub);
227 bnez(t0, stub->entry(), /* is_far */ true);
228 bind(stub->continuation());
229 }
230
231 ld(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
232
233 Label notRecursive;
234 beqz(disp_hdr, notRecursive); // Will be 0 if not recursive.
235
236 // Recursive lock
237 addi(disp_hdr, disp_hdr, -1);
238 sd(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
239 mv(flag, zr);
240 j(cont);
241
242 bind(notRecursive);
243 ld(flag, Address(tmp, ObjectMonitor::EntryList_offset()));
244 ld(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
245 orr(flag, flag, disp_hdr); // Will be 0 if both are 0.
246 bnez(flag, cont);
247 // need a release store here
248 la(tmp, Address(tmp, ObjectMonitor::owner_offset()));
249 membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
250 sd(zr, Address(tmp)); // set unowned
251
252 bind(cont);
253 // zero flag indicates success
254 // non-zero flag indicates failure
255 bnez(flag, no_count);
256
257 bind(count);
258 decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, t0, tmp);
259
260 bind(no_count);
261 }
262
263 // short string
264 // StringUTF16.indexOfChar
265 // StringLatin1.indexOfChar
266 void C2_MacroAssembler::string_indexof_char_short(Register str1, Register cnt1,
267 Register ch, Register result,
268 bool isL)
269 {
270 Register ch1 = t0;
271 Register index = t1;
272
273 BLOCK_COMMENT("string_indexof_char_short {");
274
275 Label LOOP, LOOP1, LOOP4, LOOP8;
276 Label MATCH, MATCH1, MATCH2, MATCH3,
277 MATCH4, MATCH5, MATCH6, MATCH7, NOMATCH;
278
279 mv(result, -1);
280 mv(index, zr);
|
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "asm/assembler.inline.hpp"
29 #include "opto/c2_MacroAssembler.hpp"
30 #include "opto/compile.hpp"
31 #include "opto/intrinsicnode.hpp"
32 #include "opto/output.hpp"
33 #include "opto/subnode.hpp"
34 #include "runtime/stubRoutines.hpp"
35 #include "utilities/globalDefinitions.hpp"
36
37 #ifdef PRODUCT
38 #define BLOCK_COMMENT(str) /* nothing */
39 #define STOP(error) stop(error)
40 #else
41 #define BLOCK_COMMENT(str) block_comment(str)
42 #define STOP(error) block_comment(error); stop(error)
43 #endif
44
45 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
46
47 void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg,
48 Register tmp1Reg, Register tmp2Reg, Register tmp3Reg) {
49 // Use cr register to indicate the fast_lock result: zero for success; non-zero for failure.
50 Register flag = t1;
51 Register oop = objectReg;
52 Register box = boxReg;
53 Register disp_hdr = tmp1Reg;
54 Register tmp = tmp2Reg;
55 Label object_has_monitor;
56 // Finish fast lock successfully. MUST branch to with flag == 0
57 Label locked;
58 // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
59 Label slow_path;
60
61 assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
62 assert_different_registers(oop, box, tmp, disp_hdr, flag, tmp3Reg, t0);
63
64 mv(flag, 1);
65
66 // Load markWord from object into displaced_header.
67 ld(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
68
69 if (DiagnoseSyncOnValueBasedClasses != 0) {
70 load_klass(tmp, oop);
71 lwu(tmp, Address(tmp, Klass::access_flags_offset()));
72 test_bit(tmp, tmp, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
73 bnez(tmp, slow_path);
74 }
75
76 // Check for existing monitor
77 test_bit(tmp, disp_hdr, exact_log2(markWord::monitor_value));
78 bnez(tmp, object_has_monitor);
79
80 if (LockingMode == LM_MONITOR) {
81 j(slow_path);
82 } else {
83 assert(LockingMode == LM_LEGACY, "must be");
84 // Set tmp to be (markWord of object | UNLOCK_VALUE).
85 ori(tmp, disp_hdr, markWord::unlocked_value);
86
87 // Initialize the box. (Must happen before we update the object mark!)
88 sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
89
90 // Compare object markWord with an unlocked value (tmp) and if
91 // equal exchange the stack address of our box with object markWord.
92 // On failure disp_hdr contains the possibly locked markWord.
93 cmpxchg(/*memory address*/oop, /*expected value*/tmp, /*new value*/box, Assembler::int64,
94 Assembler::aq, Assembler::rl, /*result*/disp_hdr);
95 beq(disp_hdr, tmp, locked);
96
97 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
98
99 // If the compare-and-exchange succeeded, then we found an unlocked
100 // object, will have now locked it will continue at label locked
101 // We did not see an unlocked object so try the fast recursive case.
102
103 // Check if the owner is self by comparing the value in the
104 // markWord of object (disp_hdr) with the stack pointer.
105 sub(disp_hdr, disp_hdr, sp);
106 mv(tmp, (intptr_t) (~(os::vm_page_size()-1) | (uintptr_t)markWord::lock_mask_in_place));
107 // If (mark & lock_mask) == 0 and mark - sp < page_size, we are stack-locking and goto label locked,
108 // hence we can store 0 as the displaced header in the box, which indicates that it is a
109 // recursive lock.
110 andr(tmp/*==0?*/, disp_hdr, tmp);
111 sd(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
112 beqz(tmp, locked);
113 j(slow_path);
114 }
115
116 // Handle existing monitor.
117 bind(object_has_monitor);
118 // The object's monitor m is unlocked iff m->owner == NULL,
119 // otherwise m->owner may contain a thread or a stack address.
120 //
121 // Try to CAS m->owner from NULL to current thread.
122 add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value));
123 cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/xthread, Assembler::int64,
124 Assembler::aq, Assembler::rl, /*result*/tmp3Reg); // cas succeeds if tmp3Reg == zr(expected)
125
126 // Store a non-null value into the box to avoid looking like a re-entrant
127 // lock. The fast-path monitor unlock code checks for
128 // markWord::monitor_value so use markWord::unused_mark which has the
129 // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
130 mv(tmp, (address)markWord::unused_mark().value());
131 sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
132
133 beqz(tmp3Reg, locked); // CAS success means locking succeeded
134
135 bne(tmp3Reg, xthread, slow_path); // Check for recursive locking
136
137 // Recursive lock case
138 increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1, tmp2Reg, tmp3Reg);
139
140 bind(locked);
141 mv(flag, zr);
142 increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2Reg, tmp3Reg);
143
144 #ifdef ASSERT
145 // Check that locked label is reached with flag == 0.
146 Label flag_correct;
147 beqz(flag, flag_correct);
148 stop("Fast Lock Flag != 0");
149 #endif
150
151 bind(slow_path);
152 #ifdef ASSERT
153 // Check that slow_path label is reached with flag != 0.
154 bnez(flag, flag_correct);
155 stop("Fast Lock Flag == 0");
156 bind(flag_correct);
157 #endif
158 // C2 uses the value of flag (0 vs !0) to determine the continuation.
159 }
160
161 void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg,
162 Register tmp1Reg, Register tmp2Reg) {
163 // Use cr register to indicate the fast_unlock result: zero for success; non-zero for failure.
164 Register flag = t1;
165 Register oop = objectReg;
166 Register box = boxReg;
167 Register disp_hdr = tmp1Reg;
168 Register tmp = tmp2Reg;
169 Label object_has_monitor;
170 // Finish fast lock successfully. MUST branch to with flag == 0
171 Label unlocked;
172 // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
173 Label slow_path;
174
175 assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
176 assert_different_registers(oop, box, tmp, disp_hdr, flag, t0);
177
178 mv(flag, 1);
179
180 if (LockingMode == LM_LEGACY) {
181 // Find the lock address and load the displaced header from the stack.
182 ld(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
183
184 // If the displaced header is 0, we have a recursive unlock.
185 beqz(disp_hdr, unlocked);
186 }
187
188 // Handle existing monitor.
189 ld(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
190 test_bit(t0, tmp, exact_log2(markWord::monitor_value));
191 bnez(t0, object_has_monitor);
192
193 if (LockingMode == LM_MONITOR) {
194 j(slow_path);
195 } else {
196 assert(LockingMode == LM_LEGACY, "must be");
197 // Check if it is still a light weight lock, this is true if we
198 // see the stack address of the basicLock in the markWord of the
199 // object.
200
201 cmpxchg(/*memory address*/oop, /*expected value*/box, /*new value*/disp_hdr, Assembler::int64,
202 Assembler::relaxed, Assembler::rl, /*result*/tmp);
203 beq(box, tmp, unlocked); // box == tmp if cas succeeds
204 j(slow_path);
205 }
206
207 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
208
209 // Handle existing monitor.
210 bind(object_has_monitor);
211 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
212 add(tmp, tmp, -(int)markWord::monitor_value); // monitor
213
214 ld(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
215
216 Label notRecursive;
217 beqz(disp_hdr, notRecursive); // Will be 0 if not recursive.
218
219 // Recursive lock
220 addi(disp_hdr, disp_hdr, -1);
221 sd(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
222 j(unlocked);
223
224 bind(notRecursive);
225 ld(t0, Address(tmp, ObjectMonitor::EntryList_offset()));
226 ld(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
227 orr(t0, t0, disp_hdr); // Will be 0 if both are 0.
228 bnez(t0, slow_path);
229
230 // need a release store here
231 la(tmp, Address(tmp, ObjectMonitor::owner_offset()));
232 membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
233 sd(zr, Address(tmp)); // set unowned
234
235 bind(unlocked);
236 mv(flag, zr);
237 decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp1Reg, tmp2Reg);
238
239 #ifdef ASSERT
240 // Check that unlocked label is reached with flag == 0.
241 Label flag_correct;
242 beqz(flag, flag_correct);
243 stop("Fast Lock Flag != 0");
244 #endif
245
246 bind(slow_path);
247 #ifdef ASSERT
248 // Check that slow_path label is reached with flag != 0.
249 bnez(flag, flag_correct);
250 stop("Fast Lock Flag == 0");
251 bind(flag_correct);
252 #endif
253 // C2 uses the value of flag (0 vs !0) to determine the continuation.
254 }
255
256 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register tmp1, Register tmp2, Register tmp3) {
257 // Flag register, zero for success; non-zero for failure.
258 Register flag = t1;
259
260 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
261 assert_different_registers(obj, tmp1, tmp2, tmp3, flag, t0);
262
263 mv(flag, 1);
264
265 // Handle inflated monitor.
266 Label inflated;
267 // Finish fast lock successfully. MUST branch to with flag == 0
268 Label locked;
269 // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
270 Label slow_path;
271
272 if (DiagnoseSyncOnValueBasedClasses != 0) {
273 load_klass(tmp1, obj);
274 lwu(tmp1, Address(tmp1, Klass::access_flags_offset()));
275 test_bit(tmp1, tmp1, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
276 bnez(tmp1, slow_path);
277 }
278
279 const Register tmp1_mark = tmp1;
280
281 { // Lightweight locking
282
283 // Push lock to the lock stack and finish successfully. MUST branch to with flag == 0
284 Label push;
285
286 const Register tmp2_top = tmp2;
287 const Register tmp3_t = tmp3;
288
289 // Check if lock-stack is full.
290 lwu(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
291 mv(tmp3_t, (unsigned)LockStack::end_offset());
292 bge(tmp2_top, tmp3_t, slow_path);
293
294 // Check if recursive.
295 add(tmp3_t, xthread, tmp2_top);
296 ld(tmp3_t, Address(tmp3_t, -oopSize));
297 beq(obj, tmp3_t, push);
298
299 // Relaxed normal load to check for monitor. Optimization for monitor case.
300 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
301 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
302 bnez(tmp3_t, inflated);
303
304 // Not inflated
305 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
306
307 // Try to lock. Transition lock-bits 0b01 => 0b00
308 ori(tmp1_mark, tmp1_mark, markWord::unlocked_value);
309 xori(tmp3_t, tmp1_mark, markWord::unlocked_value);
310 cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
311 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_t);
312 bne(tmp1_mark, tmp3_t, slow_path);
313
314 bind(push);
315 // After successful lock, push object on lock-stack.
316 add(tmp3_t, xthread, tmp2_top);
317 sd(obj, Address(tmp3_t));
318 addw(tmp2_top, tmp2_top, oopSize);
319 sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
320 j(locked);
321 }
322
323 { // Handle inflated monitor.
324 bind(inflated);
325
326 // mark contains the tagged ObjectMonitor*.
327 const Register tmp1_tagged_monitor = tmp1_mark;
328 const uintptr_t monitor_tag = markWord::monitor_value;
329 const Register tmp2_owner_addr = tmp2;
330 const Register tmp3_owner = tmp3;
331
332 // Compute owner address.
333 la(tmp2_owner_addr, Address(tmp1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
334
335 // CAS owner (null => current thread).
336 cmpxchg(/*addr*/ tmp2_owner_addr, /*expected*/ zr, /*new*/ xthread, Assembler::int64,
337 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_owner);
338 beqz(tmp3_owner, locked);
339
340 // Check if recursive.
341 bne(tmp3_owner, xthread, slow_path);
342
343 // Recursive.
344 increment(Address(tmp1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1, tmp2, tmp3);
345 }
346
347 bind(locked);
348 mv(flag, zr);
349 increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3);
350
351 #ifdef ASSERT
352 // Check that locked label is reached with flag == 0.
353 Label flag_correct;
354 beqz(flag, flag_correct);
355 stop("Fast Lock Flag != 0");
356 #endif
357
358 bind(slow_path);
359 #ifdef ASSERT
360 // Check that slow_path label is reached with flag != 0.
361 bnez(flag, flag_correct);
362 stop("Fast Lock Flag == 0");
363 bind(flag_correct);
364 #endif
365 // C2 uses the value of flag (0 vs !0) to determine the continuation.
366 }
367
368 void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register tmp1, Register tmp2,
369 Register tmp3) {
370 // Flag register, zero for success; non-zero for failure.
371 Register flag = t1;
372
373 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
374 assert_different_registers(obj, tmp1, tmp2, tmp3, flag, t0);
375
376 mv(flag, 1);
377
378 // Handle inflated monitor.
379 Label inflated, inflated_load_monitor;
380 // Finish fast unlock successfully. unlocked MUST branch to with flag == 0
381 Label unlocked;
382 // Finish fast unlock unsuccessfully. MUST branch to with flag != 0
383 Label slow_path;
384
385 const Register tmp1_mark = tmp1;
386 const Register tmp2_top = tmp2;
387 const Register tmp3_t = tmp3;
388
389 { // Lightweight unlock
390
391 // Check if obj is top of lock-stack.
392 lwu(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
393 subw(tmp2_top, tmp2_top, oopSize);
394 add(tmp3_t, xthread, tmp2_top);
395 ld(tmp3_t, Address(tmp3_t));
396 // Top of lock stack was not obj. Must be monitor.
397 bne(obj, tmp3_t, inflated_load_monitor);
398
399 // Pop lock-stack.
400 DEBUG_ONLY(add(tmp3_t, xthread, tmp2_top);)
401 DEBUG_ONLY(sd(zr, Address(tmp3_t));)
402 sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
403
404 // Check if recursive.
405 add(tmp3_t, xthread, tmp2_top);
406 ld(tmp3_t, Address(tmp3_t, -oopSize));
407 beq(obj, tmp3_t, unlocked);
408
409 // Not recursive.
410 // Load Mark.
411 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
412
413 // Check header for monitor (0b10).
414 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
415 bnez(tmp3_t, inflated);
416
417 // Try to unlock. Transition lock bits 0b00 => 0b01
418 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
419 ori(tmp3_t, tmp1_mark, markWord::unlocked_value);
420 cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
421 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, /*result*/ tmp3_t);
422 beq(tmp1_mark, tmp3_t, unlocked);
423
424 // Compare and exchange failed.
425 // Restore lock-stack and handle the unlock in runtime.
426 DEBUG_ONLY(add(tmp3_t, xthread, tmp2_top);)
427 DEBUG_ONLY(sd(obj, Address(tmp3_t));)
428 addw(tmp2_top, tmp2_top, oopSize);
429 sd(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
430 j(slow_path);
431 }
432
433 { // Handle inflated monitor.
434 bind(inflated_load_monitor);
435 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
436 #ifdef ASSERT
437 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
438 bnez(tmp3_t, inflated);
439 stop("Fast Unlock not monitor");
440 #endif
441
442 bind(inflated);
443
444 #ifdef ASSERT
445 Label check_done;
446 subw(tmp2_top, tmp2_top, oopSize);
447 mv(tmp3_t, in_bytes(JavaThread::lock_stack_base_offset()));
448 blt(tmp2_top, tmp3_t, check_done);
449 add(tmp3_t, xthread, tmp2_top);
450 ld(tmp3_t, Address(tmp3_t));
451 bne(obj, tmp3_t, inflated);
452 stop("Fast Unlock lock on stack");
453 bind(check_done);
454 #endif
455
456 // mark contains the tagged ObjectMonitor*.
457 const Register tmp1_monitor = tmp1_mark;
458 const uintptr_t monitor_tag = markWord::monitor_value;
459
460 // Untag the monitor.
461 sub(tmp1_monitor, tmp1_mark, monitor_tag);
462
463 const Register tmp2_recursions = tmp2;
464 Label not_recursive;
465
466 // Check if recursive.
467 ld(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
468 beqz(tmp2_recursions, not_recursive);
469
470 // Recursive unlock.
471 addi(tmp2_recursions, tmp2_recursions, -1);
472 sd(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
473 j(unlocked);
474
475 bind(not_recursive);
476
477 Label release;
478 const Register tmp2_owner_addr = tmp2;
479
480 // Compute owner address.
481 la(tmp2_owner_addr, Address(tmp1_monitor, ObjectMonitor::owner_offset()));
482
483 // Check if the entry lists are empty.
484 ld(t0, Address(tmp1_monitor, ObjectMonitor::EntryList_offset()));
485 ld(tmp3_t, Address(tmp1_monitor, ObjectMonitor::cxq_offset()));
486 orr(t0, t0, tmp3_t);
487 beqz(t0, release);
488
489 // The owner may be anonymous and we removed the last obj entry in
490 // the lock-stack. This loses the information about the owner.
491 // Write the thread to the owner field so the runtime knows the owner.
492 sd(xthread, Address(tmp2_owner_addr));
493 j(slow_path);
494
495 bind(release);
496 // Set owner to null.
497 membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
498 sd(zr, Address(tmp2_owner_addr));
499 }
500
501 bind(unlocked);
502 mv(flag, zr);
503 decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3);
504
505 #ifdef ASSERT
506 // Check that unlocked label is reached with flag == 0.
507 Label flag_correct;
508 beqz(flag, flag_correct);
509 stop("Fast Lock Flag != 0");
510 #endif
511
512 bind(slow_path);
513 #ifdef ASSERT
514 // Check that slow_path label is reached with flag != 0.
515 bnez(flag, flag_correct);
516 stop("Fast Lock Flag == 0");
517 bind(flag_correct);
518 #endif
519 // C2 uses the value of flag (0 vs !0) to determine the continuation.
520 }
521
522 // short string
523 // StringUTF16.indexOfChar
524 // StringLatin1.indexOfChar
525 void C2_MacroAssembler::string_indexof_char_short(Register str1, Register cnt1,
526 Register ch, Register result,
527 bool isL)
528 {
529 Register ch1 = t0;
530 Register index = t1;
531
532 BLOCK_COMMENT("string_indexof_char_short {");
533
534 Label LOOP, LOOP1, LOOP4, LOOP8;
535 Label MATCH, MATCH1, MATCH2, MATCH3,
536 MATCH4, MATCH5, MATCH6, MATCH7, NOMATCH;
537
538 mv(result, -1);
539 mv(index, zr);
|