1 /*
2 * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2026 SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "classfile/classLoaderData.hpp"
28 #include "gc/shared/barrierSetAssembler.hpp"
29 #include "gc/shared/barrierSetNMethod.hpp"
30 #include "gc/shared/barrierSetRuntime.hpp"
31 #include "interpreter/interp_masm.hpp"
32 #include "oops/compressedOops.hpp"
33 #include "runtime/jniHandles.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "runtime/stubRoutines.hpp"
36 #include "utilities/macros.hpp"
37 #ifdef COMPILER2
38 #include "gc/shared/c2/barrierSetC2.hpp"
39 #endif // COMPILER2
40
41 #define __ masm->
42
43 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
44 Register base, RegisterOrConstant ind_or_offs, Register val,
45 Register tmp1, Register tmp2, Register tmp3,
46 MacroAssembler::PreservationLevel preservation_level) {
47 bool in_heap = (decorators & IN_HEAP) != 0;
48 bool in_native = (decorators & IN_NATIVE) != 0;
49 bool not_null = (decorators & IS_NOT_NULL) != 0;
50 assert(in_heap || in_native, "where?");
51 assert_different_registers(base, val, tmp1, tmp2, R0);
52
53 switch (type) {
54 case T_ARRAY:
55 case T_OBJECT: {
56 if (UseCompressedOops && in_heap) {
57 Register co = tmp1;
58 if (val == noreg) {
59 assert(!not_null, "inconsistent access");
60 __ li(co, 0);
61 } else {
62 co = not_null ? __ encode_heap_oop_not_null(tmp1, val) : __ encode_heap_oop(tmp1, val);
63 }
64 __ stw(co, ind_or_offs, base, tmp2);
65 } else {
66 if (val == noreg) {
67 assert(!not_null, "inconsistent access");
68 val = tmp1;
69 __ li(val, 0);
70 }
71 __ std(val, ind_or_offs, base, tmp2);
72 }
73 break;
74 }
75 default: Unimplemented();
76 }
77 }
78
79 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
80 Register base, RegisterOrConstant ind_or_offs, Register dst,
81 Register tmp1, Register tmp2,
82 MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
83 bool in_heap = (decorators & IN_HEAP) != 0;
84 bool in_native = (decorators & IN_NATIVE) != 0;
85 bool not_null = (decorators & IS_NOT_NULL) != 0;
86 assert(in_heap || in_native, "where?");
87 assert_different_registers(ind_or_offs.register_or_noreg(), dst, R0);
88
89 switch (type) {
90 case T_ARRAY:
91 case T_OBJECT: {
92 if (UseCompressedOops && in_heap) {
93 if (L_handle_null != nullptr) { // Label provided.
94 __ lwz(dst, ind_or_offs, base);
95 __ cmpwi(CR0, dst, 0);
96 __ beq(CR0, *L_handle_null);
97 __ decode_heap_oop_not_null(dst);
98 } else if (not_null) { // Guaranteed to be not null.
99 Register narrowOop = (tmp1 != noreg && CompressedOops::base_disjoint()) ? tmp1 : dst;
100 __ lwz(narrowOop, ind_or_offs, base);
101 __ decode_heap_oop_not_null(dst, narrowOop);
102 } else { // Any oop.
103 __ lwz(dst, ind_or_offs, base);
104 __ decode_heap_oop(dst);
105 }
106 } else {
107 __ ld(dst, ind_or_offs, base);
108 if (L_handle_null != nullptr) {
109 __ cmpdi(CR0, dst, 0);
110 __ beq(CR0, *L_handle_null);
111 }
112 }
113 break;
114 }
115 default: Unimplemented();
116 }
117 }
118
119 void BarrierSetAssembler::flat_field_copy(MacroAssembler* masm, DecoratorSet decorators,
120 Register src, Register dst, Register inline_layout_info) {
121 // flat_field_copy implementation is fairly complex, and there are not any
122 // "short-cuts" to be made from asm. What there is, appears to have the same
123 // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds
124 // of hand-rolled instructions...
125 if (decorators & IS_DEST_UNINITIALIZED) {
126 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, inline_layout_info);
127 } else {
128 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, inline_layout_info);
129 }
130 }
131
132 // Generic implementation. GCs can provide an optimized one.
133 void BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value,
134 Register tmp1, Register tmp2,
135 MacroAssembler::PreservationLevel preservation_level) {
136 Label done, tagged, weak_tagged, verify;
137 __ cmpdi(CR0, value, 0);
138 __ beq(CR0, done); // Use null as-is.
139
140 __ andi_(tmp1, value, JNIHandles::tag_mask);
141 __ bne(CR0, tagged); // Test for tag.
142
143 __ access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, // no uncoloring
144 value, (intptr_t)0, value, tmp1, tmp2, preservation_level);
145 __ b(verify);
146
147 __ bind(tagged);
148 __ andi_(tmp1, value, JNIHandles::TypeTag::weak_global);
149 __ clrrdi(value, value, JNIHandles::tag_size); // Untag.
150 __ bne(CR0, weak_tagged); // Test for jweak tag.
151
152 __ access_load_at(T_OBJECT, IN_NATIVE,
153 value, (intptr_t)0, value, tmp1, tmp2, preservation_level);
154 __ b(verify);
155
156 __ bind(weak_tagged);
157 __ access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
158 value, (intptr_t)0, value, tmp1, tmp2, preservation_level);
159
160 __ bind(verify);
161 __ verify_oop(value, FILE_AND_LINE);
162 __ bind(done);
163 }
164
165 // Generic implementation. GCs can provide an optimized one.
166 void BarrierSetAssembler::resolve_global_jobject(MacroAssembler* masm, Register value,
167 Register tmp1, Register tmp2,
168 MacroAssembler::PreservationLevel preservation_level) {
169 Label done;
170
171 __ cmpdi(CR0, value, 0);
172 __ beq(CR0, done); // Use null as-is.
173
174 #ifdef ASSERT
175 {
176 Label valid_global_tag;
177 __ andi_(tmp1, value, JNIHandles::TypeTag::global);
178 __ bne(CR0, valid_global_tag); // Test for global tag.
179 __ stop("non global jobject using resolve_global_jobject");
180 __ bind(valid_global_tag);
181 }
182 #endif
183
184 __ clrrdi(value, value, JNIHandles::tag_size); // Untag.
185 __ access_load_at(T_OBJECT, IN_NATIVE,
186 value, (intptr_t)0, value, tmp1, tmp2, preservation_level);
187 __ verify_oop(value, FILE_AND_LINE);
188
189 __ bind(done);
190 }
191
192 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
193 Register obj, Register tmp, Label& slowpath) {
194 __ clrrdi(dst, obj, JNIHandles::tag_size);
195 __ ld(dst, 0, dst); // Resolve (untagged) jobject.
196 }
197
198 void BarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
199 Register tmp, Label& slow_path) {
200 // Load the oop from the weak handle without barriers.
201 __ ld(obj, 0, weak_handle);
202 }
203
204 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Register tmp) {
205 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
206 assert_different_registers(tmp, R0);
207
208 __ align(8); // must align the following block which requires atomic updates
209
210 __ block_comment("nmethod_entry_barrier (nmethod_entry_barrier) {");
211
212 // This is a compound instruction. Patching support is provided by NativeMovRegMem.
213 // Actual patching is done in (platform-specific part of) BarrierSetNMethod.
214 __ load_const32(tmp, 0 /* Value is patched */); // 2 instructions
215
216 // Low order half of 64 bit value is currently used.
217 __ ld(R0, in_bytes(bs_nm->thread_disarmed_guard_value_offset()), R16_thread);
218
219 if (TrapBasedNMethodEntryBarriers) {
220 __ tw(Assembler::traptoLessThanUnsigned | Assembler::traptoGreaterThanUnsigned, R0, tmp);
221 } else {
222 __ cmpw(CR0, R0, tmp);
223
224 // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
225 __ calculate_address_from_global_toc(tmp, StubRoutines::method_entry_barrier(),
226 true, true, false); // 2 instructions
227 __ mtctr(tmp);
228
229 __ bnectrl(CR0);
230 }
231
232 // Oops may have been changed. Make those updates observable.
233 // "isync" can serve both, data and instruction patching.
234 // But, many GCs don't modify nmethods during a concurrent phase.
235 if (nmethod_patching_type() != NMethodPatchingType::stw_instruction_and_data_patch) {
236 __ isync();
237 }
238
239 __ block_comment("} nmethod_entry_barrier (nmethod_entry_barrier)");
240 }
241
242 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler *masm, Register tmp1, Register tmp2, Register tmp3) {
243 assert_different_registers(tmp1, tmp2, tmp3);
244
245 __ block_comment("c2i_entry_barrier (c2i_entry_barrier) {");
246
247 Register tmp1_class_loader_data = tmp1;
248
249 Label bad_call, skip_barrier;
250
251 // Fast path: If no method is given, the call is definitely bad.
252 __ cmpdi(CR0, R19_method, 0);
253 __ beq(CR0, bad_call);
254
255 // Load class loader data to determine whether the method's holder is concurrently unloading.
256 __ load_method_holder(tmp1, R19_method);
257 __ ld(tmp1_class_loader_data, in_bytes(InstanceKlass::class_loader_data_offset()), tmp1);
258
259 // Fast path: If class loader is strong, the holder cannot be unloaded.
260 __ lwz(tmp2, in_bytes(ClassLoaderData::keep_alive_ref_count_offset()), tmp1_class_loader_data);
261 __ cmpdi(CR0, tmp2, 0);
262 __ bne(CR0, skip_barrier);
263
264 // Class loader is weak. Determine whether the holder is still alive.
265 __ ld(tmp2, in_bytes(ClassLoaderData::holder_offset()), tmp1_class_loader_data);
266 __ resolve_weak_handle(tmp2, tmp1, tmp3, MacroAssembler::PreservationLevel::PRESERVATION_FRAME_LR_GP_FP_REGS);
267 __ cmpdi(CR0, tmp2, 0);
268 __ bne(CR0, skip_barrier);
269
270 __ bind(bad_call);
271
272 __ calculate_address_from_global_toc(tmp1, SharedRuntime::get_handle_wrong_method_stub(), true, true, false);
273 __ mtctr(tmp1);
274 __ bctr();
275
276 __ bind(skip_barrier);
277
278 __ block_comment("} c2i_entry_barrier (c2i_entry_barrier)");
279 }
280
281 void BarrierSetAssembler::check_oop(MacroAssembler *masm, Register oop, const char* msg) {
282 __ verify_oop(oop, msg);
283 }
284
285 #ifdef COMPILER2
286
287 OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) const {
288 if (!OptoReg::is_reg(opto_reg)) {
289 return OptoReg::Bad;
290 }
291
292 VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
293 if ((vm_reg->is_Register() || vm_reg ->is_FloatRegister()) && (opto_reg & 1) != 0) {
294 return OptoReg::Bad;
295 }
296
297 return opto_reg;
298 }
299
300 #undef __
301 #define __ _masm->
302
303 SaveLiveRegisters::SaveLiveRegisters(MacroAssembler *masm, BarrierStubC2 *stub)
304 : _masm(masm), _reg_mask(stub->preserve_set()) {
305
306 const int register_save_size = iterate_over_register_mask(ACTION_COUNT_ONLY) * BytesPerWord;
307 _frame_size = align_up(register_save_size, frame::alignment_in_bytes)
308 + frame::native_abi_reg_args_size;
309
310 __ save_LR_CR(R0);
311 __ push_frame(_frame_size, R0);
312
313 iterate_over_register_mask(ACTION_SAVE, _frame_size);
314 }
315
316 SaveLiveRegisters::~SaveLiveRegisters() {
317 iterate_over_register_mask(ACTION_RESTORE, _frame_size);
318
319 __ addi(R1_SP, R1_SP, _frame_size);
320 __ restore_LR_CR(R0);
321 }
322
323 int SaveLiveRegisters::iterate_over_register_mask(IterationAction action, int offset) {
324 int reg_save_index = 0;
325 RegMaskIterator live_regs_iterator(_reg_mask);
326
327 while(live_regs_iterator.has_next()) {
328 const OptoReg::Name opto_reg = live_regs_iterator.next();
329
330 // Filter out stack slots (spilled registers, i.e., stack-allocated registers).
331 if (!OptoReg::is_reg(opto_reg)) {
332 continue;
333 }
334
335 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
336 if (vm_reg->is_Register()) {
337 Register std_reg = vm_reg->as_Register();
338
339 if (std_reg->encoding() >= R2->encoding() && std_reg->encoding() <= R12->encoding()) {
340 reg_save_index++;
341
342 if (action == ACTION_SAVE) {
343 _masm->std(std_reg, offset - reg_save_index * BytesPerWord, R1_SP);
344 } else if (action == ACTION_RESTORE) {
345 _masm->ld(std_reg, offset - reg_save_index * BytesPerWord, R1_SP);
346 } else {
347 assert(action == ACTION_COUNT_ONLY, "Sanity");
348 }
349 }
350 } else if (vm_reg->is_FloatRegister()) {
351 FloatRegister fp_reg = vm_reg->as_FloatRegister();
352 if (fp_reg->encoding() >= F0->encoding() && fp_reg->encoding() <= F13->encoding()) {
353 reg_save_index++;
354
355 if (action == ACTION_SAVE) {
356 _masm->stfd(fp_reg, offset - reg_save_index * BytesPerWord, R1_SP);
357 } else if (action == ACTION_RESTORE) {
358 _masm->lfd(fp_reg, offset - reg_save_index * BytesPerWord, R1_SP);
359 } else {
360 assert(action == ACTION_COUNT_ONLY, "Sanity");
361 }
362 }
363 } else if (vm_reg->is_ConditionRegister()) {
364 // NOP. Conditions registers are covered by save_LR_CR
365 } else if (vm_reg->is_VectorRegister()) {
366 assert(SuperwordUseVSX, "or should not reach here");
367 VectorSRegister vs_reg = (vm_reg->as_VectorRegister()).to_vsr();
368 if (vs_reg->encoding() >= VSR32->encoding() && vs_reg->encoding() <= VSR51->encoding()) {
369 reg_save_index += (2 + (reg_save_index & 1)); // 2 slots + alignment if needed
370
371 Register spill_addr = R0;
372 int spill_offset = offset - reg_save_index * BytesPerWord;
373 if (action == ACTION_SAVE) {
374 _masm->stxv(vs_reg, spill_offset, R1_SP);
375 } else if (action == ACTION_RESTORE) {
376 _masm->lxv(vs_reg, spill_offset, R1_SP);
377 } else {
378 assert(action == ACTION_COUNT_ONLY, "Sanity");
379 }
380 }
381 } else {
382 if (vm_reg->is_SpecialRegister()) {
383 fatal("Special registers are unsupported. Found register %s", vm_reg->name());
384 } else {
385 fatal("Register type is not known");
386 }
387 }
388 }
389
390 return reg_save_index;
391 }
392
393 #endif // COMPILER2