1 /*
2 * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/classLoaderData.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shared/barrierSetAssembler.hpp"
29 #include "gc/shared/barrierSetNMethod.hpp"
30 #include "gc/shared/collectedHeap.hpp"
31 #include "interpreter/interp_masm.hpp"
32 #include "memory/universe.hpp"
33 #include "runtime/javaThread.hpp"
34 #include "runtime/jniHandles.hpp"
35 #include "runtime/sharedRuntime.hpp"
36 #include "runtime/stubRoutines.hpp"
37
38
39 #define __ masm->
40
41 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
42 Register dst, Address src, Register tmp1, Register tmp2) {
43
44 // LR is live. It must be saved around calls.
45
46 bool in_heap = (decorators & IN_HEAP) != 0;
47 bool in_native = (decorators & IN_NATIVE) != 0;
48 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
49 switch (type) {
50 case T_OBJECT:
51 case T_ARRAY: {
52 if (in_heap) {
53 if (UseCompressedOops) {
54 __ ldrw(dst, src);
55 if (is_not_null) {
56 __ decode_heap_oop_not_null(dst);
57 } else {
58 __ decode_heap_oop(dst);
59 }
60 } else {
61 __ ldr(dst, src);
62 }
63 } else {
64 assert(in_native, "why else?");
65 __ ldr(dst, src);
66 }
67 break;
68 }
69 case T_BOOLEAN: __ load_unsigned_byte (dst, src); break;
70 case T_BYTE: __ load_signed_byte (dst, src); break;
71 case T_CHAR: __ load_unsigned_short(dst, src); break;
72 case T_SHORT: __ load_signed_short (dst, src); break;
73 case T_INT: __ ldrw (dst, src); break;
74 case T_LONG: __ ldr (dst, src); break;
75 case T_ADDRESS: __ ldr (dst, src); break;
76 case T_FLOAT: __ ldrs (v0, src); break;
77 case T_DOUBLE: __ ldrd (v0, src); break;
78 default: Unimplemented();
79 }
80 }
81
82 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
83 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
84 bool in_heap = (decorators & IN_HEAP) != 0;
85 bool in_native = (decorators & IN_NATIVE) != 0;
86 switch (type) {
87 case T_OBJECT:
88 case T_ARRAY: {
89 val = val == noreg ? zr : val;
90 if (in_heap) {
91 if (UseCompressedOops) {
92 assert(!dst.uses(val), "not enough registers");
93 if (val != zr) {
94 __ encode_heap_oop(val);
95 }
96 __ strw(val, dst);
97 } else {
98 __ str(val, dst);
99 }
100 } else {
101 assert(in_native, "why else?");
102 __ str(val, dst);
103 }
104 break;
105 }
106 case T_BOOLEAN:
107 __ andw(val, val, 0x1); // boolean is true if LSB is 1
108 __ strb(val, dst);
109 break;
110 case T_BYTE: __ strb(val, dst); break;
111 case T_CHAR: __ strh(val, dst); break;
112 case T_SHORT: __ strh(val, dst); break;
113 case T_INT: __ strw(val, dst); break;
114 case T_LONG: __ str (val, dst); break;
115 case T_ADDRESS: __ str (val, dst); break;
116 case T_FLOAT: __ strs(v0, dst); break;
117 case T_DOUBLE: __ strd(v0, dst); break;
118 default: Unimplemented();
119 }
120 }
121
122 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
123 DecoratorSet decorators,
124 BasicType type,
125 size_t bytes,
126 Register dst1,
127 Register dst2,
128 Address src,
129 Register tmp) {
130 if (bytes == 1) {
131 assert(dst2 == noreg, "invariant");
132 __ ldrb(dst1, src);
133 } else if (bytes == 2) {
134 assert(dst2 == noreg, "invariant");
135 __ ldrh(dst1, src);
136 } else if (bytes == 4) {
137 assert(dst2 == noreg, "invariant");
138 __ ldrw(dst1, src);
139 } else if (bytes == 8) {
140 assert(dst2 == noreg, "invariant");
141 __ ldr(dst1, src);
142 } else if (bytes == 16) {
143 assert(dst2 != noreg, "invariant");
144 assert(dst2 != dst1, "invariant");
145 __ ldp(dst1, dst2, src);
146 } else {
147 // Not the right size
148 ShouldNotReachHere();
149 }
150 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
151 __ decode_heap_oop(dst1);
152 }
153 }
154
155 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
156 DecoratorSet decorators,
157 BasicType type,
158 size_t bytes,
159 Address dst,
160 Register src1,
161 Register src2,
162 Register tmp1,
163 Register tmp2,
164 Register tmp3) {
165 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
166 __ encode_heap_oop(src1);
167 }
168 if (bytes == 1) {
169 assert(src2 == noreg, "invariant");
170 __ strb(src1, dst);
171 } else if (bytes == 2) {
172 assert(src2 == noreg, "invariant");
173 __ strh(src1, dst);
174 } else if (bytes == 4) {
175 assert(src2 == noreg, "invariant");
176 __ strw(src1, dst);
177 } else if (bytes == 8) {
178 assert(src2 == noreg, "invariant");
179 __ str(src1, dst);
180 } else if (bytes == 16) {
181 assert(src2 != noreg, "invariant");
182 assert(src2 != src1, "invariant");
183 __ stp(src1, src2, dst);
184 } else {
185 // Not the right size
186 ShouldNotReachHere();
187 }
188 }
189
190 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
191 DecoratorSet decorators,
192 BasicType type,
193 size_t bytes,
194 FloatRegister dst1,
195 FloatRegister dst2,
196 Address src,
197 Register tmp1,
198 Register tmp2,
199 FloatRegister vec_tmp) {
200 if (bytes == 32) {
201 __ ldpq(dst1, dst2, src);
202 } else {
203 ShouldNotReachHere();
204 }
205 }
206
207 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
208 DecoratorSet decorators,
209 BasicType type,
210 size_t bytes,
211 Address dst,
212 FloatRegister src1,
213 FloatRegister src2,
214 Register tmp1,
215 Register tmp2,
216 Register tmp3,
217 FloatRegister vec_tmp1,
218 FloatRegister vec_tmp2,
219 FloatRegister vec_tmp3) {
220 if (bytes == 32) {
221 __ stpq(src1, src2, dst);
222 } else {
223 ShouldNotReachHere();
224 }
225 }
226
227 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
228 Register obj, Register tmp, Label& slowpath) {
229 // If mask changes we need to ensure that the inverse is still encodable as an immediate
230 STATIC_ASSERT(JNIHandles::tag_mask == 0b11);
231 __ andr(obj, obj, ~JNIHandles::tag_mask);
232 __ ldr(obj, Address(obj, 0)); // *obj
233 }
234
235 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
236 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
237 Register var_size_in_bytes,
238 int con_size_in_bytes,
239 Register t1,
240 Register t2,
241 Label& slow_case) {
242 assert_different_registers(obj, t2);
243 assert_different_registers(obj, var_size_in_bytes);
244 Register end = t2;
245
246 // verify_tlab();
247
248 __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
249 if (var_size_in_bytes == noreg) {
250 __ lea(end, Address(obj, con_size_in_bytes));
251 } else {
252 __ lea(end, Address(obj, var_size_in_bytes));
253 }
254 __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
255 __ cmp(end, rscratch1);
256 __ br(Assembler::HI, slow_case);
257
258 // update the tlab top pointer
259 __ str(end, Address(rthread, JavaThread::tlab_top_offset()));
260
261 // recover var_size_in_bytes if necessary
262 if (var_size_in_bytes == end) {
263 __ sub(var_size_in_bytes, var_size_in_bytes, obj);
264 }
265 // verify_tlab();
266 }
267
268 void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm,
269 Register var_size_in_bytes,
270 int con_size_in_bytes,
271 Register t1) {
272 assert(t1->is_valid(), "need temp reg");
273
274 __ ldr(t1, Address(rthread, in_bytes(JavaThread::allocated_bytes_offset())));
275 if (var_size_in_bytes->is_valid()) {
276 __ add(t1, t1, var_size_in_bytes);
277 } else {
278 __ add(t1, t1, con_size_in_bytes);
279 }
280 __ str(t1, Address(rthread, in_bytes(JavaThread::allocated_bytes_offset())));
281 }
282
283 static volatile uint32_t _patching_epoch = 0;
284
285 address BarrierSetAssembler::patching_epoch_addr() {
286 return (address)&_patching_epoch;
287 }
288
289 void BarrierSetAssembler::increment_patching_epoch() {
290 Atomic::inc(&_patching_epoch);
291 }
292
293 void BarrierSetAssembler::clear_patching_epoch() {
294 _patching_epoch = 0;
295 }
296
297 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation, Label* guard) {
298 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
299
300 if (bs_nm == nullptr) {
301 return;
302 }
303
304 Label local_guard;
305 Label skip_barrier;
306 NMethodPatchingType patching_type = nmethod_patching_type();
307
308 if (slow_path == nullptr) {
309 guard = &local_guard;
310 }
311
312 // If the slow path is out of line in a stub, we flip the condition
313 Assembler::Condition condition = slow_path == nullptr ? Assembler::EQ : Assembler::NE;
314 Label& barrier_target = slow_path == nullptr ? skip_barrier : *slow_path;
315
316 __ ldrw(rscratch1, *guard);
317
318 if (patching_type == NMethodPatchingType::stw_instruction_and_data_patch) {
319 // With STW patching, no data or instructions are updated concurrently,
320 // which means there isn't really any need for any fencing for neither
321 // data nor instruction modifications happening concurrently. The
322 // instruction patching is handled with isb fences on the way back
323 // from the safepoint to Java. So here we can do a plain conditional
324 // branch with no fencing.
325 Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
326 __ ldrw(rscratch2, thread_disarmed_addr);
327 __ cmp(rscratch1, rscratch2);
328 } else if (patching_type == NMethodPatchingType::conc_instruction_and_data_patch) {
329 // If we patch code we need both a code patching and a loadload
330 // fence. It's not super cheap, so we use a global epoch mechanism
331 // to hide them in a slow path.
332 // The high level idea of the global epoch mechanism is to detect
333 // when any thread has performed the required fencing, after the
334 // last nmethod was disarmed. This implies that the required
335 // fencing has been performed for all preceding nmethod disarms
336 // as well. Therefore, we do not need any further fencing.
337 __ lea(rscratch2, ExternalAddress((address)&_patching_epoch));
338 // Embed an artificial data dependency to order the guard load
339 // before the epoch load.
340 __ orr(rscratch2, rscratch2, rscratch1, Assembler::LSR, 32);
341 // Read the global epoch value.
342 __ ldrw(rscratch2, rscratch2);
343 // Combine the guard value (low order) with the epoch value (high order).
344 __ orr(rscratch1, rscratch1, rscratch2, Assembler::LSL, 32);
345 // Compare the global values with the thread-local values.
346 Address thread_disarmed_and_epoch_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
347 __ ldr(rscratch2, thread_disarmed_and_epoch_addr);
348 __ cmp(rscratch1, rscratch2);
349 } else {
350 assert(patching_type == NMethodPatchingType::conc_data_patch, "must be");
351 // Subsequent loads of oops must occur after load of guard value.
352 // BarrierSetNMethod::disarm sets guard with release semantics.
353 __ membar(__ LoadLoad);
354 Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
355 __ ldrw(rscratch2, thread_disarmed_addr);
356 __ cmpw(rscratch1, rscratch2);
357 }
358 __ br(condition, barrier_target);
359
360 if (slow_path == nullptr) {
361 __ movptr(rscratch1, (uintptr_t) StubRoutines::aarch64::method_entry_barrier());
362 __ blr(rscratch1);
363 __ b(skip_barrier);
364
365 __ bind(local_guard);
366
367 __ emit_int32(0); // nmethod guard value. Skipped over in common case.
368 } else {
369 __ bind(*continuation);
370 }
371
372 __ bind(skip_barrier);
373 }
374
375 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
376 BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod();
377 if (bs == nullptr) {
378 return;
379 }
380
381 Label bad_call;
382 __ cbz(rmethod, bad_call);
383
384 // Pointer chase to the method holder to find out if the method is concurrently unloading.
385 Label method_live;
386 __ load_method_holder_cld(rscratch1, rmethod);
387
388 // Is it a strong CLD?
389 __ ldrw(rscratch2, Address(rscratch1, ClassLoaderData::keep_alive_offset()));
390 __ cbnz(rscratch2, method_live);
391
392 // Is it a weak but alive CLD?
393 __ push(RegSet::of(r10), sp);
394 __ ldr(r10, Address(rscratch1, ClassLoaderData::holder_offset()));
395
396 __ resolve_weak_handle(r10, rscratch1, rscratch2);
397 __ mov(rscratch1, r10);
398 __ pop(RegSet::of(r10), sp);
399 __ cbnz(rscratch1, method_live);
400
401 __ bind(bad_call);
402
403 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
404 __ bind(method_live);
405 }
406
407 void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
408 // Check if the oop is in the right area of memory
409 __ mov(tmp2, (intptr_t) Universe::verify_oop_mask());
410 __ andr(tmp1, obj, tmp2);
411 __ mov(tmp2, (intptr_t) Universe::verify_oop_bits());
412
413 // Compare tmp1 and tmp2. We don't use a compare
414 // instruction here because the flags register is live.
415 __ eor(tmp1, tmp1, tmp2);
416 __ cbnz(tmp1, error);
417
418 // make sure klass is 'reasonable', which is not zero.
419 __ load_klass(obj, obj); // get klass
420 __ cbz(obj, error); // if klass is null it is broken
421 }