1 /*
2 * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/classLoaderData.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "gc/shared/barrierSetAssembler.hpp"
28 #include "gc/shared/barrierSetNMethod.hpp"
29 #include "gc/shared/collectedHeap.hpp"
30 #include "interpreter/interp_masm.hpp"
31 #include "memory/universe.hpp"
32 #include "runtime/javaThread.hpp"
33 #include "runtime/jniHandles.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "runtime/stubRoutines.hpp"
36 #ifdef COMPILER2
37 #include "code/vmreg.inline.hpp"
38 #include "gc/shared/c2/barrierSetC2.hpp"
39 #endif // COMPILER2
40
41
42 #define __ masm->
43
44 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
45 Register dst, Address src, Register tmp1, Register tmp2) {
46
47 // LR is live. It must be saved around calls.
48
49 bool in_heap = (decorators & IN_HEAP) != 0;
50 bool in_native = (decorators & IN_NATIVE) != 0;
51 bool is_not_null = (decorators & IS_NOT_NULL) != 0;
52 switch (type) {
53 case T_OBJECT:
54 case T_ARRAY: {
55 if (in_heap) {
56 if (UseCompressedOops) {
57 __ ldrw(dst, src);
58 if (is_not_null) {
59 __ decode_heap_oop_not_null(dst);
60 } else {
61 __ decode_heap_oop(dst);
62 }
63 } else {
64 __ ldr(dst, src);
65 }
66 } else {
67 assert(in_native, "why else?");
68 __ ldr(dst, src);
69 }
70 break;
71 }
72 case T_BOOLEAN: __ load_unsigned_byte (dst, src); break;
73 case T_BYTE: __ load_signed_byte (dst, src); break;
74 case T_CHAR: __ load_unsigned_short(dst, src); break;
75 case T_SHORT: __ load_signed_short (dst, src); break;
76 case T_INT: __ ldrw (dst, src); break;
77 case T_LONG: __ ldr (dst, src); break;
78 case T_ADDRESS: __ ldr (dst, src); break;
79 case T_FLOAT: __ ldrs (v0, src); break;
80 case T_DOUBLE: __ ldrd (v0, src); break;
81 default: Unimplemented();
82 }
83 }
84
85 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
86 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
87 bool in_heap = (decorators & IN_HEAP) != 0;
88 bool in_native = (decorators & IN_NATIVE) != 0;
89 switch (type) {
90 case T_OBJECT:
91 case T_ARRAY: {
92 val = val == noreg ? zr : val;
93 if (in_heap) {
94 if (UseCompressedOops) {
95 assert(!dst.uses(val), "not enough registers");
96 if (val != zr) {
97 __ encode_heap_oop(val);
98 }
99 __ strw(val, dst);
100 } else {
101 __ str(val, dst);
102 }
103 } else {
104 assert(in_native, "why else?");
105 __ str(val, dst);
106 }
107 break;
108 }
109 case T_BOOLEAN:
110 __ andw(val, val, 0x1); // boolean is true if LSB is 1
111 __ strb(val, dst);
112 break;
113 case T_BYTE: __ strb(val, dst); break;
114 case T_CHAR: __ strh(val, dst); break;
115 case T_SHORT: __ strh(val, dst); break;
116 case T_INT: __ strw(val, dst); break;
117 case T_LONG: __ str (val, dst); break;
118 case T_ADDRESS: __ str (val, dst); break;
119 case T_FLOAT: __ strs(v0, dst); break;
120 case T_DOUBLE: __ strd(v0, dst); break;
121 default: Unimplemented();
122 }
123 }
124
125 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
126 DecoratorSet decorators,
127 BasicType type,
128 size_t bytes,
129 Register dst1,
130 Register dst2,
131 Address src,
132 Register tmp) {
133 if (bytes == 1) {
134 assert(dst2 == noreg, "invariant");
135 __ ldrb(dst1, src);
136 } else if (bytes == 2) {
137 assert(dst2 == noreg, "invariant");
138 __ ldrh(dst1, src);
139 } else if (bytes == 4) {
140 assert(dst2 == noreg, "invariant");
141 __ ldrw(dst1, src);
142 } else if (bytes == 8) {
143 assert(dst2 == noreg, "invariant");
144 __ ldr(dst1, src);
145 } else if (bytes == 16) {
146 assert(dst2 != noreg, "invariant");
147 assert(dst2 != dst1, "invariant");
148 __ ldp(dst1, dst2, src);
149 } else {
150 // Not the right size
151 ShouldNotReachHere();
152 }
153 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
154 __ decode_heap_oop(dst1);
155 }
156 }
157
158 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
159 DecoratorSet decorators,
160 BasicType type,
161 size_t bytes,
162 Address dst,
163 Register src1,
164 Register src2,
165 Register tmp1,
166 Register tmp2,
167 Register tmp3) {
168 if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
169 __ encode_heap_oop(src1);
170 }
171 if (bytes == 1) {
172 assert(src2 == noreg, "invariant");
173 __ strb(src1, dst);
174 } else if (bytes == 2) {
175 assert(src2 == noreg, "invariant");
176 __ strh(src1, dst);
177 } else if (bytes == 4) {
178 assert(src2 == noreg, "invariant");
179 __ strw(src1, dst);
180 } else if (bytes == 8) {
181 assert(src2 == noreg, "invariant");
182 __ str(src1, dst);
183 } else if (bytes == 16) {
184 assert(src2 != noreg, "invariant");
185 assert(src2 != src1, "invariant");
186 __ stp(src1, src2, dst);
187 } else {
188 // Not the right size
189 ShouldNotReachHere();
190 }
191 }
192
193 void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
194 DecoratorSet decorators,
195 BasicType type,
196 size_t bytes,
197 FloatRegister dst1,
198 FloatRegister dst2,
199 Address src,
200 Register tmp1,
201 Register tmp2,
202 FloatRegister vec_tmp) {
203 if (bytes == 32) {
204 __ ldpq(dst1, dst2, src);
205 } else {
206 ShouldNotReachHere();
207 }
208 }
209
210 void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
211 DecoratorSet decorators,
212 BasicType type,
213 size_t bytes,
214 Address dst,
215 FloatRegister src1,
216 FloatRegister src2,
217 Register tmp1,
218 Register tmp2,
219 Register tmp3,
220 FloatRegister vec_tmp1,
221 FloatRegister vec_tmp2,
222 FloatRegister vec_tmp3) {
223 if (bytes == 32) {
224 __ stpq(src1, src2, dst);
225 } else {
226 ShouldNotReachHere();
227 }
228 }
229
230 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
231 Register obj, Register tmp, Label& slowpath) {
232 // If mask changes we need to ensure that the inverse is still encodable as an immediate
233 STATIC_ASSERT(JNIHandles::tag_mask == 0b11);
234 __ andr(obj, obj, ~JNIHandles::tag_mask);
235 __ ldr(obj, Address(obj, 0)); // *obj
236 }
237
238 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
239 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
240 Register var_size_in_bytes,
241 int con_size_in_bytes,
242 Register t1,
243 Register t2,
244 Label& slow_case) {
245 assert_different_registers(obj, t2);
246 assert_different_registers(obj, var_size_in_bytes);
247 Register end = t2;
248
249 // verify_tlab();
250
251 __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
252 if (var_size_in_bytes == noreg) {
253 __ lea(end, Address(obj, con_size_in_bytes));
254 } else {
255 __ lea(end, Address(obj, var_size_in_bytes));
256 }
257 __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
258 __ cmp(end, rscratch1);
259 __ br(Assembler::HI, slow_case);
260
261 // update the tlab top pointer
262 __ str(end, Address(rthread, JavaThread::tlab_top_offset()));
263
264 // recover var_size_in_bytes if necessary
265 if (var_size_in_bytes == end) {
266 __ sub(var_size_in_bytes, var_size_in_bytes, obj);
267 }
268 // verify_tlab();
269 }
270
271 static volatile uint32_t _patching_epoch = 0;
272
273 address BarrierSetAssembler::patching_epoch_addr() {
274 return (address)&_patching_epoch;
275 }
276
277 void BarrierSetAssembler::increment_patching_epoch() {
278 AtomicAccess::inc(&_patching_epoch);
279 }
280
281 void BarrierSetAssembler::clear_patching_epoch() {
282 _patching_epoch = 0;
283 }
284
285 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation, Label* guard) {
286 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
287
288 Label local_guard;
289 Label skip_barrier;
290 NMethodPatchingType patching_type = nmethod_patching_type();
291
292 if (slow_path == nullptr) {
293 guard = &local_guard;
294 }
295
296 // If the slow path is out of line in a stub, we flip the condition
297 Assembler::Condition condition = slow_path == nullptr ? Assembler::EQ : Assembler::NE;
298 Label& barrier_target = slow_path == nullptr ? skip_barrier : *slow_path;
299
300 __ ldrw(rscratch1, *guard);
301
302 if (patching_type == NMethodPatchingType::stw_instruction_and_data_patch) {
303 // With STW patching, no data or instructions are updated concurrently,
304 // which means there isn't really any need for any fencing for neither
305 // data nor instruction modifications happening concurrently. The
306 // instruction patching is handled with isb fences on the way back
307 // from the safepoint to Java. So here we can do a plain conditional
308 // branch with no fencing.
309 Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
310 __ ldrw(rscratch2, thread_disarmed_addr);
311 __ cmp(rscratch1, rscratch2);
312 } else if (patching_type == NMethodPatchingType::conc_instruction_and_data_patch) {
313 // If we patch code we need both a code patching and a loadload
314 // fence. It's not super cheap, so we use a global epoch mechanism
315 // to hide them in a slow path.
316 // The high level idea of the global epoch mechanism is to detect
317 // when any thread has performed the required fencing, after the
318 // last nmethod was disarmed. This implies that the required
319 // fencing has been performed for all preceding nmethod disarms
320 // as well. Therefore, we do not need any further fencing.
321 __ lea(rscratch2, ExternalAddress((address)&_patching_epoch));
322 // Embed an artificial data dependency to order the guard load
323 // before the epoch load.
324 __ orr(rscratch2, rscratch2, rscratch1, Assembler::LSR, 32);
325 // Read the global epoch value.
326 __ ldrw(rscratch2, rscratch2);
327 // Combine the guard value (low order) with the epoch value (high order).
328 __ orr(rscratch1, rscratch1, rscratch2, Assembler::LSL, 32);
329 // Compare the global values with the thread-local values.
330 Address thread_disarmed_and_epoch_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
331 __ ldr(rscratch2, thread_disarmed_and_epoch_addr);
332 __ cmp(rscratch1, rscratch2);
333 } else {
334 ShouldNotReachHere();
335 }
336 __ br(condition, barrier_target);
337
338 if (slow_path == nullptr) {
339 __ lea(rscratch1, RuntimeAddress(StubRoutines::method_entry_barrier()));
340 __ blr(rscratch1);
341 __ b(skip_barrier);
342
343 __ bind(local_guard);
344
345 __ emit_int32(0); // nmethod guard value. Skipped over in common case.
346 } else {
347 __ bind(*continuation);
348 }
349
350 __ bind(skip_barrier);
351 }
352
353 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
354 Label bad_call;
355 __ cbz(rmethod, bad_call);
356
357 // Pointer chase to the method holder to find out if the method is concurrently unloading.
358 Label method_live;
359 __ load_method_holder_cld(rscratch1, rmethod);
360
361 // Is it a strong CLD?
362 __ ldrw(rscratch2, Address(rscratch1, ClassLoaderData::keep_alive_ref_count_offset()));
363 __ cbnz(rscratch2, method_live);
364
365 // Is it a weak but alive CLD?
366 __ push(RegSet::of(r10), sp);
367 __ ldr(r10, Address(rscratch1, ClassLoaderData::holder_offset()));
368
369 __ resolve_weak_handle(r10, rscratch1, rscratch2);
370 __ mov(rscratch1, r10);
371 __ pop(RegSet::of(r10), sp);
372 __ cbnz(rscratch1, method_live);
373
374 __ bind(bad_call);
375
376 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
377 __ bind(method_live);
378 }
379
380 void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
381 // Check if the oop is in the right area of memory
382 __ mov(tmp2, (intptr_t) Universe::verify_oop_mask());
383 __ andr(tmp1, obj, tmp2);
384 __ mov(tmp2, (intptr_t) Universe::verify_oop_bits());
385
386 // Compare tmp1 and tmp2. We don't use a compare
387 // instruction here because the flags register is live.
388 __ eor(tmp1, tmp1, tmp2);
389 __ cbnz(tmp1, error);
390
391 // make sure klass is 'reasonable', which is not zero.
392 __ load_klass(obj, obj); // get klass
393 __ cbz(obj, error); // if klass is null it is broken
394 }
395
396 #ifdef COMPILER2
397
398 OptoReg::Name BarrierSetAssembler::encode_float_vector_register_size(const Node* node, OptoReg::Name opto_reg) {
399 switch (node->ideal_reg()) {
400 case Op_RegF:
401 case Op_RegI: // RA may place scalar values (Op_RegI/N/L/P) in FP registers when UseFPUForSpilling is enabled
402 case Op_RegN:
403 // No need to refine. The original encoding is already fine to distinguish.
404 assert(opto_reg % 4 == 0, "32-bit register should only occupy a single slot");
405 break;
406 // Use different encoding values of the same fp/vector register to help distinguish different sizes.
407 // Such as V16. The OptoReg::name and its corresponding slot value are
408 // "V16": 64, "V16_H": 65, "V16_J": 66, "V16_K": 67.
409 case Op_RegD:
410 case Op_VecD:
411 case Op_RegL:
412 case Op_RegP:
413 opto_reg &= ~3;
414 opto_reg |= 1;
415 break;
416 case Op_VecX:
417 opto_reg &= ~3;
418 opto_reg |= 2;
419 break;
420 case Op_VecA:
421 opto_reg &= ~3;
422 opto_reg |= 3;
423 break;
424 default:
425 assert(false, "unexpected ideal register");
426 ShouldNotReachHere();
427 }
428 return opto_reg;
429 }
430
431 OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
432 if (!OptoReg::is_reg(opto_reg)) {
433 return OptoReg::Bad;
434 }
435
436 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
437 if (vm_reg->is_FloatRegister()) {
438 opto_reg = encode_float_vector_register_size(node, opto_reg);
439 }
440
441 return opto_reg;
442 }
443
444 void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
445 // Load the oop from the weak handle.
446 __ ldr(obj, Address(obj));
447 }
448
449 #undef __
450 #define __ _masm->
451
452 void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
453 int index = -1;
454 GrowableArray<RegisterData> registers;
455 VMReg prev_vm_reg = VMRegImpl::Bad();
456
457 RegMaskIterator rmi(stub->preserve_set());
458 while (rmi.has_next()) {
459 OptoReg::Name opto_reg = rmi.next();
460 VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
461
462 if (vm_reg->is_Register()) {
463 // GPR may have one or two slots in regmask
464 // Determine whether the current vm_reg is the same physical register as the previous one
465 if (is_same_register(vm_reg, prev_vm_reg)) {
466 registers.at(index)._slots++;
467 } else {
468 RegisterData reg_data = { vm_reg, 1 };
469 index = registers.append(reg_data);
470 }
471 } else if (vm_reg->is_FloatRegister()) {
472 // We have size encoding in OptoReg of stub->preserve_set()
473 // After encoding, float/neon/sve register has only one slot in regmask
474 // Decode it to get the actual size
475 VMReg vm_reg_base = vm_reg->as_FloatRegister()->as_VMReg();
476 int slots = decode_float_vector_register_size(opto_reg);
477 RegisterData reg_data = { vm_reg_base, slots };
478 index = registers.append(reg_data);
479 } else if (vm_reg->is_PRegister()) {
480 // PRegister has only one slot in regmask
481 RegisterData reg_data = { vm_reg, 1 };
482 index = registers.append(reg_data);
483 } else {
484 assert(false, "Unknown register type");
485 ShouldNotReachHere();
486 }
487 prev_vm_reg = vm_reg;
488 }
489
490 // Record registers that needs to be saved/restored
491 for (GrowableArrayIterator<RegisterData> it = registers.begin(); it != registers.end(); ++it) {
492 RegisterData reg_data = *it;
493 VMReg vm_reg = reg_data._reg;
494 int slots = reg_data._slots;
495 if (vm_reg->is_Register()) {
496 assert(slots == 1 || slots == 2, "Unexpected register save size");
497 _gp_regs += RegSet::of(vm_reg->as_Register());
498 } else if (vm_reg->is_FloatRegister()) {
499 if (slots == 1 || slots == 2) {
500 _fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
501 } else if (slots == 4) {
502 _neon_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
503 } else {
504 assert(slots == Matcher::scalable_vector_reg_size(T_FLOAT), "Unexpected register save size");
505 _sve_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
506 }
507 } else {
508 assert(vm_reg->is_PRegister() && slots == 1, "Unknown register type");
509 _p_regs += PRegSet::of(vm_reg->as_PRegister());
510 }
511 }
512
513 // Remove C-ABI SOE registers and scratch regs
514 _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9);
515
516 // Remove C-ABI SOE fp registers
517 _fp_regs -= FloatRegSet::range(v8, v15);
518 }
519
520 enum RC SaveLiveRegisters::rc_class(VMReg reg) {
521 if (reg->is_reg()) {
522 if (reg->is_Register()) {
523 return rc_int;
524 } else if (reg->is_FloatRegister()) {
525 return rc_float;
526 } else if (reg->is_PRegister()) {
527 return rc_predicate;
528 }
529 }
530 if (reg->is_stack()) {
531 return rc_stack;
532 }
533 return rc_bad;
534 }
535
536 bool SaveLiveRegisters::is_same_register(VMReg reg1, VMReg reg2) {
537 if (reg1 == reg2) {
538 return true;
539 }
540 if (rc_class(reg1) == rc_class(reg2)) {
541 if (reg1->is_Register()) {
542 return reg1->as_Register() == reg2->as_Register();
543 } else if (reg1->is_FloatRegister()) {
544 return reg1->as_FloatRegister() == reg2->as_FloatRegister();
545 } else if (reg1->is_PRegister()) {
546 return reg1->as_PRegister() == reg2->as_PRegister();
547 }
548 }
549 return false;
550 }
551
552 int SaveLiveRegisters::decode_float_vector_register_size(OptoReg::Name opto_reg) {
553 switch (opto_reg & 3) {
554 case 0:
555 return 1;
556 case 1:
557 return 2;
558 case 2:
559 return 4;
560 case 3:
561 return Matcher::scalable_vector_reg_size(T_FLOAT);
562 default:
563 ShouldNotReachHere();
564 return 0;
565 }
566 }
567
568 SaveLiveRegisters::SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub)
569 : _masm(masm),
570 _gp_regs(),
571 _fp_regs(),
572 _neon_regs(),
573 _sve_regs(),
574 _p_regs() {
575
576 // Figure out what registers to save/restore
577 initialize(stub);
578
579 // Save registers
580 __ push(_gp_regs, sp);
581 __ push_fp(_fp_regs, sp, MacroAssembler::PushPopFp);
582 __ push_fp(_neon_regs, sp, MacroAssembler::PushPopNeon);
583 __ push_fp(_sve_regs, sp, MacroAssembler::PushPopSVE);
584 __ push_p(_p_regs, sp);
585 }
586
587 SaveLiveRegisters::~SaveLiveRegisters() {
588 // Restore registers
589 __ pop_p(_p_regs, sp);
590 __ pop_fp(_sve_regs, sp, MacroAssembler::PushPopSVE);
591 __ pop_fp(_neon_regs, sp, MacroAssembler::PushPopNeon);
592 __ pop_fp(_fp_regs, sp, MacroAssembler::PushPopFp);
593
594 // External runtime call may clobber ptrue reg
595 __ reinitialize_ptrue();
596
597 __ pop(_gp_regs, sp);
598 }
599
600 #endif // COMPILER2