1 /*
2 * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/classLoaderData.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shared/barrierSetAssembler.hpp"
29 #include "gc/shared/barrierSetNMethod.hpp"
30 #include "gc/shared/barrierSetRuntime.hpp"
31 #include "gc/shared/collectedHeap.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "memory/universe.hpp"
34 #include "runtime/javaThread.hpp"
35 #include "runtime/jniHandles.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "runtime/stubRoutines.hpp"
38
39
40 #define __ masm->
41
42 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
43 Register dst, Address src, Register tmp1, Register tmp2) {
44
45 // LR is live. It must be saved around calls.
46
47 bool in_heap = (decorators & IN_HEAP) != 0;
48 bool in_native = (decorators & IN_NATIVE) != 0;
49 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
50
51 switch (type) {
52 case T_OBJECT:
53 case T_ARRAY: {
54 if (in_heap) {
55 if (UseCompressedOops) {
56 __ ldrw(dst, src);
57 if (is_not_null) {
58 __ decode_heap_oop_not_null(dst);
59 } else {
60 __ decode_heap_oop(dst);
61 }
62 } else {
63 __ ldr(dst, src);
64 }
65 } else {
66 assert(in_native, "why else?");
67 __ ldr(dst, src);
68 }
69 break;
70 }
71 case T_BOOLEAN: __ load_unsigned_byte (dst, src); break;
72 case T_BYTE: __ load_signed_byte (dst, src); break;
73 case T_CHAR: __ load_unsigned_short(dst, src); break;
74 case T_SHORT: __ load_signed_short (dst, src); break;
75 case T_INT: __ ldrw (dst, src); break;
76 case T_LONG: __ ldr (dst, src); break;
77 case T_ADDRESS: __ ldr (dst, src); break;
78 case T_FLOAT: __ ldrs (v0, src); break;
79 case T_DOUBLE: __ ldrd (v0, src); break;
80 default: Unimplemented();
81 }
82 }
83
84 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
85 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
86 bool in_heap = (decorators & IN_HEAP) != 0;
87 bool in_native = (decorators & IN_NATIVE) != 0;
88 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
89
90 switch (type) {
91 case T_OBJECT:
92 case T_ARRAY: {
93 if (in_heap) {
94 if (val == noreg) {
95 assert(!is_not_null, "inconsistent access");
96 if (UseCompressedOops) {
97 __ strw(zr, dst);
98 } else {
99 __ str(zr, dst);
100 }
101 } else {
102 if (UseCompressedOops) {
103 assert(!dst.uses(val), "not enough registers");
104 if (is_not_null) {
105 __ encode_heap_oop_not_null(val);
106 } else {
107 __ encode_heap_oop(val);
108 }
109 __ strw(val, dst);
110 } else {
111 __ str(val, dst);
112 }
113 }
114 } else {
115 assert(in_native, "why else?");
116 assert(val != noreg, "not supported");
117 __ str(val, dst);
118 }
119 break;
120 }
121 case T_BOOLEAN:
122 __ andw(val, val, 0x1); // boolean is true if LSB is 1
123 __ strb(val, dst);
124 break;
125 case T_BYTE: __ strb(val, dst); break;
126 case T_CHAR: __ strh(val, dst); break;
127 case T_SHORT: __ strh(val, dst); break;
128 case T_INT: __ strw(val, dst); break;
129 case T_LONG: __ str (val, dst); break;
130 case T_ADDRESS: __ str (val, dst); break;
131 case T_FLOAT: __ strs(v0, dst); break;
132 case T_DOUBLE: __ strd(v0, dst); break;
133 default: Unimplemented();
134 }
135 }
136
137 void BarrierSetAssembler::value_copy(MacroAssembler* masm, DecoratorSet decorators,
138 Register src, Register dst, Register value_klass) {
139 // value_copy implementation is fairly complex, and there are not any
140 // "short-cuts" to be made from asm. What there is, appears to have the same
141 // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds
142 // of hand-rolled instructions...
143 if (decorators & IS_DEST_UNINITIALIZED) {
144 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, value_klass);
145 } else {
146 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, value_klass);
147 }
148 }
149
150 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
151 DecoratorSet decorators,
152 BasicType type,
153 size_t bytes,
154 Register dst1,
155 Register dst2,
156 Address src,
157 Register tmp) {
158 if (bytes == 1) {
159 assert(dst2 == noreg, "invariant");
160 __ ldrb(dst1, src);
161 } else if (bytes == 2) {
162 assert(dst2 == noreg, "invariant");
163 __ ldrh(dst1, src);
164 } else if (bytes == 4) {
165 assert(dst2 == noreg, "invariant");
166 __ ldrw(dst1, src);
167 } else if (bytes == 8) {
168 assert(dst2 == noreg, "invariant");
169 __ ldr(dst1, src);
170 } else if (bytes == 16) {
171 assert(dst2 != noreg, "invariant");
172 assert(dst2 != dst1, "invariant");
173 __ ldp(dst1, dst2, src);
174 } else {
175 // Not the right size
176 ShouldNotReachHere();
177 }
178 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
179 __ decode_heap_oop(dst1);
180 }
181 }
182
183 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
184 DecoratorSet decorators,
185 BasicType type,
186 size_t bytes,
187 Address dst,
188 Register src1,
189 Register src2,
190 Register tmp1,
191 Register tmp2,
192 Register tmp3) {
193 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
194 __ encode_heap_oop(src1);
195 }
196 if (bytes == 1) {
197 assert(src2 == noreg, "invariant");
198 __ strb(src1, dst);
199 } else if (bytes == 2) {
200 assert(src2 == noreg, "invariant");
201 __ strh(src1, dst);
202 } else if (bytes == 4) {
203 assert(src2 == noreg, "invariant");
204 __ strw(src1, dst);
205 } else if (bytes == 8) {
206 assert(src2 == noreg, "invariant");
207 __ str(src1, dst);
208 } else if (bytes == 16) {
209 assert(src2 != noreg, "invariant");
210 assert(src2 != src1, "invariant");
211 __ stp(src1, src2, dst);
212 } else {
213 // Not the right size
214 ShouldNotReachHere();
215 }
216 }
217
218 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
219 DecoratorSet decorators,
220 BasicType type,
221 size_t bytes,
222 FloatRegister dst1,
223 FloatRegister dst2,
224 Address src,
225 Register tmp1,
226 Register tmp2,
227 FloatRegister vec_tmp) {
228 if (bytes == 32) {
229 __ ldpq(dst1, dst2, src);
230 } else {
231 ShouldNotReachHere();
232 }
233 }
234
235 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
236 DecoratorSet decorators,
237 BasicType type,
238 size_t bytes,
239 Address dst,
240 FloatRegister src1,
241 FloatRegister src2,
242 Register tmp1,
243 Register tmp2,
244 Register tmp3,
245 FloatRegister vec_tmp1,
246 FloatRegister vec_tmp2,
247 FloatRegister vec_tmp3) {
248 if (bytes == 32) {
249 __ stpq(src1, src2, dst);
250 } else {
251 ShouldNotReachHere();
252 }
253 }
254
255 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
256 Register obj, Register tmp, Label& slowpath) {
257 // If mask changes we need to ensure that the inverse is still encodable as an immediate
258 STATIC_ASSERT(JNIHandles::tag_mask == 0b11);
259 __ andr(obj, obj, ~JNIHandles::tag_mask);
260 __ ldr(obj, Address(obj, 0)); // *obj
261 }
262
263 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
264 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
265 Register var_size_in_bytes,
266 int con_size_in_bytes,
267 Register t1,
268 Register t2,
269 Label& slow_case) {
270 assert_different_registers(obj, t2);
271 assert_different_registers(obj, var_size_in_bytes);
272 Register end = t2;
273
274 // verify_tlab();
275
276 __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
277 if (var_size_in_bytes == noreg) {
278 __ lea(end, Address(obj, con_size_in_bytes));
279 } else {
280 __ lea(end, Address(obj, var_size_in_bytes));
281 }
282 __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
283 __ cmp(end, rscratch1);
284 __ br(Assembler::HI, slow_case);
285
286 // update the tlab top pointer
287 __ str(end, Address(rthread, JavaThread::tlab_top_offset()));
288
289 // recover var_size_in_bytes if necessary
290 if (var_size_in_bytes == end) {
291 __ sub(var_size_in_bytes, var_size_in_bytes, obj);
292 }
293 // verify_tlab();
294 }
295
296 void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm,
297 Register var_size_in_bytes,
298 int con_size_in_bytes,
299 Register t1) {
300 assert(t1->is_valid(), "need temp reg");
301
302 __ ldr(t1, Address(rthread, in_bytes(JavaThread::allocated_bytes_offset())));
303 if (var_size_in_bytes->is_valid()) {
304 __ add(t1, t1, var_size_in_bytes);
305 } else {
306 __ add(t1, t1, con_size_in_bytes);
307 }
308 __ str(t1, Address(rthread, in_bytes(JavaThread::allocated_bytes_offset())));
309 }
310
311 static volatile uint32_t _patching_epoch = 0;
312
313 address BarrierSetAssembler::patching_epoch_addr() {
314 return (address)&_patching_epoch;
315 }
316
317 void BarrierSetAssembler::increment_patching_epoch() {
318 Atomic::inc(&_patching_epoch);
319 }
320
321 void BarrierSetAssembler::clear_patching_epoch() {
322 _patching_epoch = 0;
323 }
324
325 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation, Label* guard) {
326 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
327
328 if (bs_nm == nullptr) {
329 return;
330 }
331
332 Label local_guard;
333 Label skip_barrier;
334 NMethodPatchingType patching_type = nmethod_patching_type();
335
336 if (slow_path == nullptr) {
337 guard = &local_guard;
338 }
339
340 // If the slow path is out of line in a stub, we flip the condition
341 Assembler::Condition condition = slow_path == nullptr ? Assembler::EQ : Assembler::NE;
342 Label& barrier_target = slow_path == nullptr ? skip_barrier : *slow_path;
343
344 __ ldrw(rscratch1, *guard);
345
346 if (patching_type == NMethodPatchingType::stw_instruction_and_data_patch) {
347 // With STW patching, no data or instructions are updated concurrently,
348 // which means there isn't really any need for any fencing for neither
349 // data nor instruction modifications happening concurrently. The
350 // instruction patching is handled with isb fences on the way back
351 // from the safepoint to Java. So here we can do a plain conditional
352 // branch with no fencing.
353 Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
354 __ ldrw(rscratch2, thread_disarmed_addr);
355 __ cmp(rscratch1, rscratch2);
356 } else if (patching_type == NMethodPatchingType::conc_instruction_and_data_patch) {
357 // If we patch code we need both a code patching and a loadload
358 // fence. It's not super cheap, so we use a global epoch mechanism
359 // to hide them in a slow path.
360 // The high level idea of the global epoch mechanism is to detect
361 // when any thread has performed the required fencing, after the
362 // last nmethod was disarmed. This implies that the required
363 // fencing has been performed for all preceding nmethod disarms
364 // as well. Therefore, we do not need any further fencing.
365 __ lea(rscratch2, ExternalAddress((address)&_patching_epoch));
366 // Embed an artificial data dependency to order the guard load
367 // before the epoch load.
368 __ orr(rscratch2, rscratch2, rscratch1, Assembler::LSR, 32);
369 // Read the global epoch value.
370 __ ldrw(rscratch2, rscratch2);
371 // Combine the guard value (low order) with the epoch value (high order).
372 __ orr(rscratch1, rscratch1, rscratch2, Assembler::LSL, 32);
373 // Compare the global values with the thread-local values.
374 Address thread_disarmed_and_epoch_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
375 __ ldr(rscratch2, thread_disarmed_and_epoch_addr);
376 __ cmp(rscratch1, rscratch2);
377 } else {
378 assert(patching_type == NMethodPatchingType::conc_data_patch, "must be");
379 // Subsequent loads of oops must occur after load of guard value.
380 // BarrierSetNMethod::disarm sets guard with release semantics.
381 __ membar(__ LoadLoad);
382 Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
383 __ ldrw(rscratch2, thread_disarmed_addr);
384 __ cmpw(rscratch1, rscratch2);
385 }
386 __ br(condition, barrier_target);
387
388 if (slow_path == nullptr) {
389 __ lea(rscratch1, RuntimeAddress(StubRoutines::method_entry_barrier()));
390 __ blr(rscratch1);
391 __ b(skip_barrier);
392
393 __ bind(local_guard);
394
395 __ emit_int32(0); // nmethod guard value. Skipped over in common case.
396 } else {
397 __ bind(*continuation);
398 }
399
400 __ bind(skip_barrier);
401 }
402
403 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
404 BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod();
405 if (bs == nullptr) {
406 return;
407 }
408
409 Label bad_call;
410 __ cbz(rmethod, bad_call);
411
412 // Pointer chase to the method holder to find out if the method is concurrently unloading.
413 Label method_live;
414 __ load_method_holder_cld(rscratch1, rmethod);
415
416 // Is it a strong CLD?
417 __ ldrw(rscratch2, Address(rscratch1, ClassLoaderData::keep_alive_offset()));
418 __ cbnz(rscratch2, method_live);
419
420 // Is it a weak but alive CLD?
421 __ push(RegSet::of(r10), sp);
422 __ ldr(r10, Address(rscratch1, ClassLoaderData::holder_offset()));
423
424 __ resolve_weak_handle(r10, rscratch1, rscratch2);
425 __ mov(rscratch1, r10);
426 __ pop(RegSet::of(r10), sp);
427 __ cbnz(rscratch1, method_live);
428
429 __ bind(bad_call);
430
431 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
432 __ bind(method_live);
433 }
434
435 void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
436 // Check if the oop is in the right area of memory
437 __ mov(tmp2, (intptr_t) Universe::verify_oop_mask());
438 __ andr(tmp1, obj, tmp2);
439 __ mov(tmp2, (intptr_t) Universe::verify_oop_bits());
440
441 // Compare tmp1 and tmp2. We don't use a compare
442 // instruction here because the flags register is live.
443 __ eor(tmp1, tmp1, tmp2);
444 __ cbnz(tmp1, error);
445
446 // make sure klass is 'reasonable', which is not zero.
447 __ load_klass(obj, obj); // get klass
448 __ cbz(obj, error); // if klass is null it is broken
449 }