15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "opto/c2_MacroAssembler.hpp"
29 #include "opto/compile.hpp"
30 #include "opto/intrinsicnode.hpp"
31 #include "opto/matcher.hpp"
32 #include "opto/output.hpp"
33 #include "opto/subnode.hpp"
34 #include "runtime/stubRoutines.hpp"
35
36 #ifdef PRODUCT
37 #define BLOCK_COMMENT(str) /* nothing */
38 #define STOP(error) stop(error)
39 #else
40 #define BLOCK_COMMENT(str) block_comment(str)
41 #define STOP(error) block_comment(error); stop(error)
42 #endif
43
44 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
45
46 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
47
48 void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register tmpReg,
49 Register tmp2Reg, Register tmp3Reg) {
50 Register oop = objectReg;
51 Register box = boxReg;
52 Register disp_hdr = tmpReg;
53 Register tmp = tmp2Reg;
54 Label cont;
55 Label object_has_monitor;
56 Label count, no_count;
57
58 assert_different_registers(oop, box, tmp, disp_hdr);
59
60 // Load markWord from object into displaced_header.
61 ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
62
63 if (DiagnoseSyncOnValueBasedClasses != 0) {
64 load_klass(tmp, oop);
65 ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
66 tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
67 br(Assembler::NE, cont);
68 }
69
70 // Check for existing monitor
71 tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
72
73 if (LockingMode == LM_MONITOR) {
74 tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
75 b(cont);
76 } else if (LockingMode == LM_LEGACY) {
77 // Set tmp to be (markWord of object | UNLOCK_VALUE).
78 orr(tmp, disp_hdr, markWord::unlocked_value);
79
80 // Initialize the box. (Must happen before we update the object mark!)
81 str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
82
83 // Compare object markWord with an unlocked value (tmp) and if
84 // equal exchange the stack address of our box with object markWord.
85 // On failure disp_hdr contains the possibly locked markWord.
86 cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
87 /*release*/ true, /*weak*/ false, disp_hdr);
88 br(Assembler::EQ, cont);
89
90 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
91
92 // If the compare-and-exchange succeeded, then we found an unlocked
93 // object, will have now locked it will continue at label cont
94
95 // Check if the owner is self by comparing the value in the
96 // markWord of object (disp_hdr) with the stack pointer.
97 mov(rscratch1, sp);
98 sub(disp_hdr, disp_hdr, rscratch1);
99 mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
100 // If condition is true we are cont and hence we can store 0 as the
101 // displaced header in the box, which indicates that it is a recursive lock.
102 ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
103 str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
104 b(cont);
105 } else {
106 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
107 lightweight_lock(oop, disp_hdr, tmp, tmp3Reg, no_count);
108 b(count);
109 }
110
111 // Handle existing monitor.
112 bind(object_has_monitor);
113
114 // The object's monitor m is unlocked iff m->owner == NULL,
115 // otherwise m->owner may contain a thread or a stack address.
116 //
117 // Try to CAS m->owner from NULL to current thread.
118 add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
119 cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
120 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
121
122 if (LockingMode != LM_LIGHTWEIGHT) {
123 // Store a non-null value into the box to avoid looking like a re-entrant
124 // lock. The fast-path monitor unlock code checks for
125 // markWord::monitor_value so use markWord::unused_mark which has the
126 // relevant bit set, and also matches ObjectSynchronizer::enter.
127 mov(tmp, (address)markWord::unused_mark().value());
128 str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
129 }
130 br(Assembler::EQ, cont); // CAS success means locking succeeded
131
132 cmp(rscratch1, rthread);
133 br(Assembler::NE, cont); // Check for recursive locking
134
135 // Recursive lock case
136 increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
137 // flag == EQ still from the cmp above, checking if this is a reentrant lock
138
139 bind(cont);
140 // flag == EQ indicates success
141 // flag == NE indicates failure
142 br(Assembler::NE, no_count);
143
144 bind(count);
145 increment(Address(rthread, JavaThread::held_monitor_count_offset()));
146
147 bind(no_count);
148 }
149
150 void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Register tmpReg,
151 Register tmp2Reg) {
152 Register oop = objectReg;
153 Register box = boxReg;
154 Register disp_hdr = tmpReg;
155 Register tmp = tmp2Reg;
156 Label cont;
157 Label object_has_monitor;
158 Label count, no_count;
159
160 assert_different_registers(oop, box, tmp, disp_hdr);
161
162 if (LockingMode == LM_LEGACY) {
163 // Find the lock address and load the displaced header from the stack.
164 ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
165
166 // If the displaced header is 0, we have a recursive unlock.
167 cmp(disp_hdr, zr);
168 br(Assembler::EQ, cont);
169 }
170
171 // Handle existing monitor.
172 ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
173 tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
174
175 if (LockingMode == LM_MONITOR) {
176 tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
177 b(cont);
178 } else if (LockingMode == LM_LEGACY) {
179 // Check if it is still a light weight lock, this is is true if we
180 // see the stack address of the basicLock in the markWord of the
181 // object.
182
183 cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
184 /*release*/ true, /*weak*/ false, tmp);
185 b(cont);
186 } else {
187 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
188 lightweight_unlock(oop, tmp, box, disp_hdr, no_count);
189 b(count);
190 }
191
192 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
193
194 // Handle existing monitor.
195 bind(object_has_monitor);
196 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
197 add(tmp, tmp, -(int)markWord::monitor_value); // monitor
198
199 if (LockingMode == LM_LIGHTWEIGHT) {
200 // If the owner is anonymous, we need to fix it -- in an outline stub.
201 Register tmp2 = disp_hdr;
202 ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
203 // We cannot use tbnz here, the target might be too far away and cannot
204 // be encoded.
205 tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
206 C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
207 Compile::current()->output()->add_stub(stub);
208 br(Assembler::NE, stub->entry());
209 bind(stub->continuation());
210 }
211
212 ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
213
214 Label notRecursive;
215 cbz(disp_hdr, notRecursive);
216
217 // Recursive lock
218 sub(disp_hdr, disp_hdr, 1u);
219 str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
220 cmp(disp_hdr, disp_hdr); // Sets flags for result
221 b(cont);
222
223 bind(notRecursive);
224 ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
225 ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
226 orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
227 cmp(rscratch1, zr); // Sets flags for result
228 cbnz(rscratch1, cont);
229 // need a release store here
230 lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
231 stlr(zr, tmp); // set unowned
232
233 bind(cont);
234 // flag == EQ indicates success
235 // flag == NE indicates failure
236 br(Assembler::NE, no_count);
237
238 bind(count);
239 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
240
241 bind(no_count);
242 }
243
244 // Search for str1 in str2 and return index or -1
245 // Clobbers: rscratch1, rscratch2, rflags. May also clobber v0-v1, when icnt1==-1.
246 void C2_MacroAssembler::string_indexof(Register str2, Register str1,
247 Register cnt2, Register cnt1,
248 Register tmp1, Register tmp2,
249 Register tmp3, Register tmp4,
250 Register tmp5, Register tmp6,
251 int icnt1, Register result, int ae) {
252 // NOTE: tmp5, tmp6 can be zr depending on specific method version
253 Label LINEARSEARCH, LINEARSTUB, LINEAR_MEDIUM, DONE, NOMATCH, MATCH;
254
255 Register ch1 = rscratch1;
256 Register ch2 = rscratch2;
257 Register cnt1tmp = tmp1;
258 Register cnt2tmp = tmp2;
259 Register cnt1_neg = cnt1;
260 Register cnt2_neg = cnt2;
261 Register result_tmp = tmp4;
262
263 bool isL = ae == StrIntrinsicNode::LL;
2237 sve_and(vtmp, T, min_jlong);
2238 sve_orr(vtmp, T, jlong_cast(1.0));
2239 break;
2240 default:
2241 assert(false, "unsupported");
2242 ShouldNotReachHere();
2243 }
2244 sve_sel(dst, T, pgtmp, vtmp, src); // Select either from src or vtmp based on the predicate register pgtmp
2245 // Result in dst
2246 }
2247
2248 bool C2_MacroAssembler::in_scratch_emit_size() {
2249 if (ciEnv::current()->task() != nullptr) {
2250 PhaseOutput* phase_output = Compile::current()->output();
2251 if (phase_output != nullptr && phase_output->in_scratch_emit_size()) {
2252 return true;
2253 }
2254 }
2255 return MacroAssembler::in_scratch_emit_size();
2256 }
|
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "opto/c2_MacroAssembler.hpp"
29 #include "opto/compile.hpp"
30 #include "opto/intrinsicnode.hpp"
31 #include "opto/matcher.hpp"
32 #include "opto/output.hpp"
33 #include "opto/subnode.hpp"
34 #include "runtime/stubRoutines.hpp"
35 #include "utilities/globalDefinitions.hpp"
36
37 #ifdef PRODUCT
38 #define BLOCK_COMMENT(str) /* nothing */
39 #define STOP(error) stop(error)
40 #else
41 #define BLOCK_COMMENT(str) block_comment(str)
42 #define STOP(error) block_comment(error); stop(error)
43 #endif
44
45 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
46
47 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
48
49 void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register tmpReg,
50 Register tmp2Reg, Register tmp3Reg) {
51 Register oop = objectReg;
52 Register box = boxReg;
53 Register disp_hdr = tmpReg;
54 Register tmp = tmp2Reg;
55 Label cont;
56 Label object_has_monitor;
57 Label count, no_count;
58
59 assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
60 assert_different_registers(oop, box, tmp, disp_hdr);
61
62 // Load markWord from object into displaced_header.
63 ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
64
65 if (DiagnoseSyncOnValueBasedClasses != 0) {
66 load_klass(tmp, oop);
67 ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
68 tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
69 br(Assembler::NE, cont);
70 }
71
72 // Check for existing monitor
73 tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
74
75 if (LockingMode == LM_MONITOR) {
76 tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
77 b(cont);
78 } else {
79 assert(LockingMode == LM_LEGACY, "must be");
80 // Set tmp to be (markWord of object | UNLOCK_VALUE).
81 orr(tmp, disp_hdr, markWord::unlocked_value);
82
83 // Initialize the box. (Must happen before we update the object mark!)
84 str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
85
86 // Compare object markWord with an unlocked value (tmp) and if
87 // equal exchange the stack address of our box with object markWord.
88 // On failure disp_hdr contains the possibly locked markWord.
89 cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
90 /*release*/ true, /*weak*/ false, disp_hdr);
91 br(Assembler::EQ, cont);
92
93 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
94
95 // If the compare-and-exchange succeeded, then we found an unlocked
96 // object, will have now locked it will continue at label cont
97
98 // Check if the owner is self by comparing the value in the
99 // markWord of object (disp_hdr) with the stack pointer.
100 mov(rscratch1, sp);
101 sub(disp_hdr, disp_hdr, rscratch1);
102 mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
103 // If condition is true we are cont and hence we can store 0 as the
104 // displaced header in the box, which indicates that it is a recursive lock.
105 ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
106 str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
107 b(cont);
108 }
109
110 // Handle existing monitor.
111 bind(object_has_monitor);
112
113 // The object's monitor m is unlocked iff m->owner == NULL,
114 // otherwise m->owner may contain a thread or a stack address.
115 //
116 // Try to CAS m->owner from NULL to current thread.
117 add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
118 cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
119 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
120
121 // Store a non-null value into the box to avoid looking like a re-entrant
122 // lock. The fast-path monitor unlock code checks for
123 // markWord::monitor_value so use markWord::unused_mark which has the
124 // relevant bit set, and also matches ObjectSynchronizer::enter.
125 mov(tmp, (address)markWord::unused_mark().value());
126 str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
127
128 br(Assembler::EQ, cont); // CAS success means locking succeeded
129
130 cmp(rscratch1, rthread);
131 br(Assembler::NE, cont); // Check for recursive locking
132
133 // Recursive lock case
134 increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
135 // flag == EQ still from the cmp above, checking if this is a reentrant lock
136
137 bind(cont);
138 // flag == EQ indicates success
139 // flag == NE indicates failure
140 br(Assembler::NE, no_count);
141
142 bind(count);
143 increment(Address(rthread, JavaThread::held_monitor_count_offset()));
144
145 bind(no_count);
146 }
147
148 void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Register tmpReg,
149 Register tmp2Reg) {
150 Register oop = objectReg;
151 Register box = boxReg;
152 Register disp_hdr = tmpReg;
153 Register tmp = tmp2Reg;
154 Label cont;
155 Label object_has_monitor;
156 Label count, no_count;
157
158 assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
159 assert_different_registers(oop, box, tmp, disp_hdr);
160
161 if (LockingMode == LM_LEGACY) {
162 // Find the lock address and load the displaced header from the stack.
163 ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
164
165 // If the displaced header is 0, we have a recursive unlock.
166 cmp(disp_hdr, zr);
167 br(Assembler::EQ, cont);
168 }
169
170 // Handle existing monitor.
171 ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
172 tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
173
174 if (LockingMode == LM_MONITOR) {
175 tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
176 b(cont);
177 } else {
178 assert(LockingMode == LM_LEGACY, "must be");
179 // Check if it is still a light weight lock, this is is true if we
180 // see the stack address of the basicLock in the markWord of the
181 // object.
182
183 cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
184 /*release*/ true, /*weak*/ false, tmp);
185 b(cont);
186 }
187
188 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
189
190 // Handle existing monitor.
191 bind(object_has_monitor);
192 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
193 add(tmp, tmp, -(int)markWord::monitor_value); // monitor
194
195 ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
196
197 Label notRecursive;
198 cbz(disp_hdr, notRecursive);
199
200 // Recursive lock
201 sub(disp_hdr, disp_hdr, 1u);
202 str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
203 cmp(disp_hdr, disp_hdr); // Sets flags for result
204 b(cont);
205
206 bind(notRecursive);
207 ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
208 ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
209 orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
210 cmp(rscratch1, zr); // Sets flags for result
211 cbnz(rscratch1, cont);
212 // need a release store here
213 lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
214 stlr(zr, tmp); // set unowned
215
216 bind(cont);
217 // flag == EQ indicates success
218 // flag == NE indicates failure
219 br(Assembler::NE, no_count);
220
221 bind(count);
222 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
223
224 bind(no_count);
225 }
226
227 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
228 Register t2, Register t3) {
229 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
230 assert_different_registers(obj, t1, t2, t3);
231
232 // Handle inflated monitor.
233 Label inflated;
234 // Finish fast lock successfully. MUST branch to with flag == EQ
235 Label locked;
236 // Finish fast lock unsuccessfully. MUST branch to with flag == NE
237 Label slow_path;
238
239 if (DiagnoseSyncOnValueBasedClasses != 0) {
240 load_klass(t1, obj);
241 ldrw(t1, Address(t1, Klass::access_flags_offset()));
242 tstw(t1, JVM_ACC_IS_VALUE_BASED_CLASS);
243 br(Assembler::NE, slow_path);
244 }
245
246 const Register t1_mark = t1;
247
248 { // Lightweight locking
249
250 // Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
251 Label push;
252
253 const Register t2_top = t2;
254 const Register t3_t = t3;
255
256 // Check if lock-stack is full.
257 ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
258 cmpw(t2_top, (unsigned)LockStack::end_offset() - 1);
259 br(Assembler::GT, slow_path);
260
261 // Check if recursive.
262 subw(t3_t, t2_top, oopSize);
263 ldr(t3_t, Address(rthread, t3_t));
264 cmp(obj, t3_t);
265 br(Assembler::EQ, push);
266
267 // Relaxed normal load to check for monitor. Optimization for monitor case.
268 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
269 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
270
271 // Not inflated
272 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
273
274 // Try to lock. Transition lock-bits 0b01 => 0b00
275 orr(t1_mark, t1_mark, markWord::unlocked_value);
276 eor(t3_t, t1_mark, markWord::unlocked_value);
277 cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
278 /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
279 br(Assembler::NE, slow_path);
280
281 bind(push);
282 // After successful lock, push object on lock-stack.
283 str(obj, Address(rthread, t2_top));
284 addw(t2_top, t2_top, oopSize);
285 strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
286 b(locked);
287 }
288
289 { // Handle inflated monitor.
290 bind(inflated);
291
292 // mark contains the tagged ObjectMonitor*.
293 const Register t1_tagged_monitor = t1_mark;
294 const uintptr_t monitor_tag = markWord::monitor_value;
295 const Register t2_owner_addr = t2;
296 const Register t3_owner = t3;
297
298 // Compute owner address.
299 lea(t2_owner_addr, Address(t1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
300
301 // CAS owner (null => current thread).
302 cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
303 /*release*/ false, /*weak*/ false, t3_owner);
304 br(Assembler::EQ, locked);
305
306 // Check if recursive.
307 cmp(t3_owner, rthread);
308 br(Assembler::NE, slow_path);
309
310 // Recursive.
311 increment(Address(t1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1);
312 }
313
314 bind(locked);
315 increment(Address(rthread, JavaThread::held_monitor_count_offset()));
316
317 #ifdef ASSERT
318 // Check that locked label is reached with Flags == EQ.
319 Label flag_correct;
320 br(Assembler::EQ, flag_correct);
321 stop("Fast Lock Flag != EQ");
322 #endif
323
324 bind(slow_path);
325 #ifdef ASSERT
326 // Check that slow_path label is reached with Flags == NE.
327 br(Assembler::NE, flag_correct);
328 stop("Fast Lock Flag != NE");
329 bind(flag_correct);
330 #endif
331 // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
332 }
333
334 void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Register t2,
335 Register t3) {
336 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
337 assert_different_registers(obj, t1, t2, t3);
338
339 // Handle inflated monitor.
340 Label inflated, inflated_load_monitor;
341 // Finish fast unlock successfully. MUST branch to with flag == EQ
342 Label unlocked;
343 // Finish fast unlock unsuccessfully. MUST branch to with flag == NE
344 Label slow_path;
345
346 const Register t1_mark = t1;
347 const Register t2_top = t2;
348 const Register t3_t = t3;
349
350 { // Lightweight unlock
351
352 // Check if obj is top of lock-stack.
353 ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
354 subw(t2_top, t2_top, oopSize);
355 ldr(t3_t, Address(rthread, t2_top));
356 cmp(obj, t3_t);
357 // Top of lock stack was not obj. Must be monitor.
358 br(Assembler::NE, inflated_load_monitor);
359
360 // Pop lock-stack.
361 DEBUG_ONLY(str(zr, Address(rthread, t2_top));)
362 strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
363
364 // Check if recursive.
365 subw(t3_t, t2_top, oopSize);
366 ldr(t3_t, Address(rthread, t3_t));
367 cmp(obj, t3_t);
368 br(Assembler::EQ, unlocked);
369
370 // Not recursive.
371 // Load Mark.
372 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
373
374 // Check header for monitor (0b10).
375 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
376
377 // Try to unlock. Transition lock bits 0b00 => 0b01
378 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
379 orr(t3_t, t1_mark, markWord::unlocked_value);
380 cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
381 /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
382 br(Assembler::EQ, unlocked);
383
384 // Compare and exchange failed.
385 // Restore lock-stack and handle the unlock in runtime.
386 DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
387 addw(t2_top, t2_top, oopSize);
388 str(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
389 b(slow_path);
390 }
391
392
393 { // Handle inflated monitor.
394 bind(inflated_load_monitor);
395 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
396 #ifdef ASSERT
397 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
398 stop("Fast Unlock not monitor");
399 #endif
400
401 bind(inflated);
402
403 #ifdef ASSERT
404 Label check_done;
405 subw(t2_top, t2_top, oopSize);
406 cmpw(t2_top, in_bytes(JavaThread::lock_stack_base_offset()));
407 br(Assembler::LT, check_done);
408 ldr(t3_t, Address(rthread, t2_top));
409 cmp(obj, t3_t);
410 br(Assembler::NE, inflated);
411 stop("Fast Unlock lock on stack");
412 bind(check_done);
413 #endif
414
415 // mark contains the tagged ObjectMonitor*.
416 const Register t1_monitor = t1_mark;
417 const uintptr_t monitor_tag = markWord::monitor_value;
418
419 // Untag the monitor.
420 sub(t1_monitor, t1_mark, monitor_tag);
421
422 const Register t2_recursions = t2;
423 Label not_recursive;
424
425 // Check if recursive.
426 ldr(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
427 cbz(t2_recursions, not_recursive);
428
429 // Recursive unlock.
430 sub(t2_recursions, t2_recursions, 1u);
431 str(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
432 // Set flag == EQ
433 cmp(t2_recursions, t2_recursions);
434 b(unlocked);
435
436 bind(not_recursive);
437
438 Label release;
439 const Register t2_owner_addr = t2;
440
441 // Compute owner address.
442 lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
443
444 // Check if the entry lists are empty.
445 ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
446 ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
447 orr(rscratch1, rscratch1, t3_t);
448 cmp(rscratch1, zr);
449 br(Assembler::EQ, release);
450
451 // The owner may be anonymous and we removed the last obj entry in
452 // the lock-stack. This loses the information about the owner.
453 // Write the thread to the owner field so the runtime knows the owner.
454 str(rthread, Address(t2_owner_addr));
455 b(slow_path);
456
457 bind(release);
458 // Set owner to null.
459 // Release to satisfy the JMM
460 stlr(zr, t2_owner_addr);
461 }
462
463 bind(unlocked);
464 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
465
466 #ifdef ASSERT
467 // Check that unlocked label is reached with Flags == EQ.
468 Label flag_correct;
469 br(Assembler::EQ, flag_correct);
470 stop("Fast Unlock Flag != EQ");
471 #endif
472
473 bind(slow_path);
474 #ifdef ASSERT
475 // Check that slow_path label is reached with Flags == NE.
476 br(Assembler::NE, flag_correct);
477 stop("Fast Unlock Flag != NE");
478 bind(flag_correct);
479 #endif
480 // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
481 }
482
483 // Search for str1 in str2 and return index or -1
484 // Clobbers: rscratch1, rscratch2, rflags. May also clobber v0-v1, when icnt1==-1.
485 void C2_MacroAssembler::string_indexof(Register str2, Register str1,
486 Register cnt2, Register cnt1,
487 Register tmp1, Register tmp2,
488 Register tmp3, Register tmp4,
489 Register tmp5, Register tmp6,
490 int icnt1, Register result, int ae) {
491 // NOTE: tmp5, tmp6 can be zr depending on specific method version
492 Label LINEARSEARCH, LINEARSTUB, LINEAR_MEDIUM, DONE, NOMATCH, MATCH;
493
494 Register ch1 = rscratch1;
495 Register ch2 = rscratch2;
496 Register cnt1tmp = tmp1;
497 Register cnt2tmp = tmp2;
498 Register cnt1_neg = cnt1;
499 Register cnt2_neg = cnt2;
500 Register result_tmp = tmp4;
501
502 bool isL = ae == StrIntrinsicNode::LL;
2476 sve_and(vtmp, T, min_jlong);
2477 sve_orr(vtmp, T, jlong_cast(1.0));
2478 break;
2479 default:
2480 assert(false, "unsupported");
2481 ShouldNotReachHere();
2482 }
2483 sve_sel(dst, T, pgtmp, vtmp, src); // Select either from src or vtmp based on the predicate register pgtmp
2484 // Result in dst
2485 }
2486
2487 bool C2_MacroAssembler::in_scratch_emit_size() {
2488 if (ciEnv::current()->task() != nullptr) {
2489 PhaseOutput* phase_output = Compile::current()->output();
2490 if (phase_output != nullptr && phase_output->in_scratch_emit_size()) {
2491 return true;
2492 }
2493 }
2494 return MacroAssembler::in_scratch_emit_size();
2495 }
2496
2497 void C2_MacroAssembler::load_nklass_compact(Register dst, Register obj, Register index, int scale, int disp) {
2498 C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
2499 Compile::current()->output()->add_stub(stub);
2500
2501 // Note: Don't clobber obj anywhere in that method!
2502
2503 // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
2504 // obj-start, so that we can load from the object's mark-word instead. Usually the address
2505 // comes as obj-start in obj and klass_offset_in_bytes in disp. However, sometimes C2
2506 // emits code that pre-computes obj-start + klass_offset_in_bytes into a register, and
2507 // then passes that register as obj and 0 in disp. The following code extracts the base
2508 // and offset to load the mark-word.
2509 int offset = oopDesc::mark_offset_in_bytes() + disp - oopDesc::klass_offset_in_bytes();
2510 if (index == noreg) {
2511 ldr(dst, Address(obj, offset));
2512 } else {
2513 lea(dst, Address(obj, index, Address::lsl(scale)));
2514 ldr(dst, Address(dst, offset));
2515 }
2516 // NOTE: We can't use tbnz here, because the target is sometimes too far away
2517 // and cannot be encoded.
2518 tst(dst, markWord::monitor_value);
2519 br(Assembler::NE, stub->entry());
2520 bind(stub->continuation());
2521 lsr(dst, dst, markWord::klass_shift);
2522 }
|