1 /*
2 * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2026 SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "classfile/classLoaderData.hpp"
28 #include "gc/shared/barrierSetAssembler.hpp"
29 #include "gc/shared/barrierSetNMethod.hpp"
30 #include "interpreter/interp_masm.hpp"
31 #include "oops/compressedOops.hpp"
32 #include "runtime/jniHandles.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "runtime/stubRoutines.hpp"
35 #include "utilities/macros.hpp"
36 #ifdef COMPILER2
37 #include "gc/shared/c2/barrierSetC2.hpp"
38 #endif // COMPILER2
39
40 #define __ masm->
41
42 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
43 Register base, RegisterOrConstant ind_or_offs, Register val,
44 Register tmp1, Register tmp2, Register tmp3,
45 MacroAssembler::PreservationLevel preservation_level) {
46 bool in_heap = (decorators & IN_HEAP) != 0;
47 bool in_native = (decorators & IN_NATIVE) != 0;
48 bool not_null = (decorators & IS_NOT_NULL) != 0;
49 assert(in_heap || in_native, "where?");
50 assert_different_registers(base, val, tmp1, tmp2, R0);
51
52 switch (type) {
53 case T_ARRAY:
54 case T_OBJECT: {
55 if (UseCompressedOops && in_heap) {
56 Register co = tmp1;
57 if (val == noreg) {
58 __ li(co, 0);
59 } else {
60 co = not_null ? __ encode_heap_oop_not_null(tmp1, val) : __ encode_heap_oop(tmp1, val);
61 }
62 __ stw(co, ind_or_offs, base, tmp2);
63 } else {
64 if (val == noreg) {
65 val = tmp1;
66 __ li(val, 0);
67 }
68 __ std(val, ind_or_offs, base, tmp2);
69 }
70 break;
71 }
72 default: Unimplemented();
73 }
74 }
75
76 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
77 Register base, RegisterOrConstant ind_or_offs, Register dst,
78 Register tmp1, Register tmp2,
79 MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
80 bool in_heap = (decorators & IN_HEAP) != 0;
81 bool in_native = (decorators & IN_NATIVE) != 0;
82 bool not_null = (decorators & IS_NOT_NULL) != 0;
83 assert(in_heap || in_native, "where?");
84 assert_different_registers(ind_or_offs.register_or_noreg(), dst, R0);
85
86 switch (type) {
87 case T_ARRAY:
88 case T_OBJECT: {
89 if (UseCompressedOops && in_heap) {
90 if (L_handle_null != nullptr) { // Label provided.
91 __ lwz(dst, ind_or_offs, base);
92 __ cmpwi(CR0, dst, 0);
93 __ beq(CR0, *L_handle_null);
94 __ decode_heap_oop_not_null(dst);
95 } else if (not_null) { // Guaranteed to be not null.
96 Register narrowOop = (tmp1 != noreg && CompressedOops::base_disjoint()) ? tmp1 : dst;
97 __ lwz(narrowOop, ind_or_offs, base);
98 __ decode_heap_oop_not_null(dst, narrowOop);
99 } else { // Any oop.
100 __ lwz(dst, ind_or_offs, base);
101 __ decode_heap_oop(dst);
102 }
103 } else {
104 __ ld(dst, ind_or_offs, base);
105 if (L_handle_null != nullptr) {
106 __ cmpdi(CR0, dst, 0);
107 __ beq(CR0, *L_handle_null);
108 }
109 }
110 break;
111 }
112 default: Unimplemented();
113 }
114 }
115
116 // Generic implementation. GCs can provide an optimized one.
117 void BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value,
118 Register tmp1, Register tmp2,
119 MacroAssembler::PreservationLevel preservation_level) {
120 Label done, tagged, weak_tagged, verify;
121 __ cmpdi(CR0, value, 0);
122 __ beq(CR0, done); // Use null as-is.
123
124 __ andi_(tmp1, value, JNIHandles::tag_mask);
125 __ bne(CR0, tagged); // Test for tag.
126
127 __ access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, // no uncoloring
128 value, (intptr_t)0, value, tmp1, tmp2, preservation_level);
129 __ b(verify);
130
131 __ bind(tagged);
132 __ andi_(tmp1, value, JNIHandles::TypeTag::weak_global);
133 __ clrrdi(value, value, JNIHandles::tag_size); // Untag.
134 __ bne(CR0, weak_tagged); // Test for jweak tag.
135
136 __ access_load_at(T_OBJECT, IN_NATIVE,
137 value, (intptr_t)0, value, tmp1, tmp2, preservation_level);
138 __ b(verify);
139
140 __ bind(weak_tagged);
141 __ access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
142 value, (intptr_t)0, value, tmp1, tmp2, preservation_level);
143
144 __ bind(verify);
145 __ verify_oop(value, FILE_AND_LINE);
146 __ bind(done);
147 }
148
149 // Generic implementation. GCs can provide an optimized one.
150 void BarrierSetAssembler::resolve_global_jobject(MacroAssembler* masm, Register value,
151 Register tmp1, Register tmp2,
152 MacroAssembler::PreservationLevel preservation_level) {
153 Label done;
154
155 __ cmpdi(CR0, value, 0);
156 __ beq(CR0, done); // Use null as-is.
157
158 #ifdef ASSERT
159 {
160 Label valid_global_tag;
161 __ andi_(tmp1, value, JNIHandles::TypeTag::global);
162 __ bne(CR0, valid_global_tag); // Test for global tag.
163 __ stop("non global jobject using resolve_global_jobject");
164 __ bind(valid_global_tag);
165 }
166 #endif
167
168 __ clrrdi(value, value, JNIHandles::tag_size); // Untag.
169 __ access_load_at(T_OBJECT, IN_NATIVE,
170 value, (intptr_t)0, value, tmp1, tmp2, preservation_level);
171 __ verify_oop(value, FILE_AND_LINE);
172
173 __ bind(done);
174 }
175
176 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
177 Register obj, Register tmp, Label& slowpath) {
178 __ clrrdi(dst, obj, JNIHandles::tag_size);
179 __ ld(dst, 0, dst); // Resolve (untagged) jobject.
180 }
181
182 void BarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
183 Register tmp, Label& slow_path) {
184 // Load the oop from the weak handle without barriers.
185 __ ld(obj, 0, weak_handle);
186 }
187
188 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Register tmp) {
189 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
190 assert_different_registers(tmp, R0);
191
192 __ align(8); // must align the following block which requires atomic updates
193
194 __ block_comment("nmethod_entry_barrier (nmethod_entry_barrier) {");
195
196 // This is a compound instruction. Patching support is provided by NativeMovRegMem.
197 // Actual patching is done in (platform-specific part of) BarrierSetNMethod.
198 __ load_const32(tmp, 0 /* Value is patched */); // 2 instructions
199
200 // Low order half of 64 bit value is currently used.
201 __ ld(R0, in_bytes(bs_nm->thread_disarmed_guard_value_offset()), R16_thread);
202
203 if (TrapBasedNMethodEntryBarriers) {
204 __ tw(Assembler::traptoLessThanUnsigned | Assembler::traptoGreaterThanUnsigned, R0, tmp);
205 } else {
206 __ cmpw(CR0, R0, tmp);
207
208 // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
209 __ calculate_address_from_global_toc(tmp, StubRoutines::method_entry_barrier(),
210 true, true, false); // 2 instructions
211 __ mtctr(tmp);
212
213 __ bnectrl(CR0);
214 }
215
216 // Oops may have been changed. Make those updates observable.
217 // "isync" can serve both, data and instruction patching.
218 // But, many GCs don't modify nmethods during a concurrent phase.
219 if (nmethod_patching_type() != NMethodPatchingType::stw_instruction_and_data_patch) {
220 __ isync();
221 }
222
223 __ block_comment("} nmethod_entry_barrier (nmethod_entry_barrier)");
224 }
225
226 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler *masm, Register tmp1, Register tmp2, Register tmp3) {
227 assert_different_registers(tmp1, tmp2, tmp3);
228
229 __ block_comment("c2i_entry_barrier (c2i_entry_barrier) {");
230
231 Register tmp1_class_loader_data = tmp1;
232
233 Label bad_call, skip_barrier;
234
235 // Fast path: If no method is given, the call is definitely bad.
236 __ cmpdi(CR0, R19_method, 0);
237 __ beq(CR0, bad_call);
238
239 // Load class loader data to determine whether the method's holder is concurrently unloading.
240 __ load_method_holder(tmp1, R19_method);
241 __ ld(tmp1_class_loader_data, in_bytes(InstanceKlass::class_loader_data_offset()), tmp1);
242
243 // Fast path: If class loader is strong, the holder cannot be unloaded.
244 __ lwz(tmp2, in_bytes(ClassLoaderData::keep_alive_ref_count_offset()), tmp1_class_loader_data);
245 __ cmpdi(CR0, tmp2, 0);
246 __ bne(CR0, skip_barrier);
247
248 // Class loader is weak. Determine whether the holder is still alive.
249 __ ld(tmp2, in_bytes(ClassLoaderData::holder_offset()), tmp1_class_loader_data);
250 __ resolve_weak_handle(tmp2, tmp1, tmp3, MacroAssembler::PreservationLevel::PRESERVATION_FRAME_LR_GP_FP_REGS);
251 __ cmpdi(CR0, tmp2, 0);
252 __ bne(CR0, skip_barrier);
253
254 __ bind(bad_call);
255
256 __ calculate_address_from_global_toc(tmp1, SharedRuntime::get_handle_wrong_method_stub(), true, true, false);
257 __ mtctr(tmp1);
258 __ bctr();
259
260 __ bind(skip_barrier);
261
262 __ block_comment("} c2i_entry_barrier (c2i_entry_barrier)");
263 }
264
265 void BarrierSetAssembler::check_oop(MacroAssembler *masm, Register oop, const char* msg) {
266 __ verify_oop(oop, msg);
267 }
268
269 #ifdef COMPILER2
270
271 OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) const {
272 if (!OptoReg::is_reg(opto_reg)) {
273 return OptoReg::Bad;
274 }
275
276 VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
277 if ((vm_reg->is_Register() || vm_reg ->is_FloatRegister()) && (opto_reg & 1) != 0) {
278 return OptoReg::Bad;
279 }
280
281 return opto_reg;
282 }
283
284 #undef __
285 #define __ _masm->
286
287 SaveLiveRegisters::SaveLiveRegisters(MacroAssembler *masm, BarrierStubC2 *stub)
288 : _masm(masm), _reg_mask(stub->preserve_set()) {
289
290 const int register_save_size = iterate_over_register_mask(ACTION_COUNT_ONLY) * BytesPerWord;
291 _frame_size = align_up(register_save_size, frame::alignment_in_bytes)
292 + frame::native_abi_reg_args_size;
293
294 __ save_LR_CR(R0);
295 __ push_frame(_frame_size, R0);
296
297 iterate_over_register_mask(ACTION_SAVE, _frame_size);
298 }
299
300 SaveLiveRegisters::~SaveLiveRegisters() {
301 iterate_over_register_mask(ACTION_RESTORE, _frame_size);
302
303 __ addi(R1_SP, R1_SP, _frame_size);
304 __ restore_LR_CR(R0);
305 }
306
307 int SaveLiveRegisters::iterate_over_register_mask(IterationAction action, int offset) {
308 int reg_save_index = 0;
309 RegMaskIterator live_regs_iterator(_reg_mask);
310
311 while(live_regs_iterator.has_next()) {
312 const OptoReg::Name opto_reg = live_regs_iterator.next();
313
314 // Filter out stack slots (spilled registers, i.e., stack-allocated registers).
315 if (!OptoReg::is_reg(opto_reg)) {
316 continue;
317 }
318
319 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
320 if (vm_reg->is_Register()) {
321 Register std_reg = vm_reg->as_Register();
322
323 if (std_reg->encoding() >= R2->encoding() && std_reg->encoding() <= R12->encoding()) {
324 reg_save_index++;
325
326 if (action == ACTION_SAVE) {
327 _masm->std(std_reg, offset - reg_save_index * BytesPerWord, R1_SP);
328 } else if (action == ACTION_RESTORE) {
329 _masm->ld(std_reg, offset - reg_save_index * BytesPerWord, R1_SP);
330 } else {
331 assert(action == ACTION_COUNT_ONLY, "Sanity");
332 }
333 }
334 } else if (vm_reg->is_FloatRegister()) {
335 FloatRegister fp_reg = vm_reg->as_FloatRegister();
336 if (fp_reg->encoding() >= F0->encoding() && fp_reg->encoding() <= F13->encoding()) {
337 reg_save_index++;
338
339 if (action == ACTION_SAVE) {
340 _masm->stfd(fp_reg, offset - reg_save_index * BytesPerWord, R1_SP);
341 } else if (action == ACTION_RESTORE) {
342 _masm->lfd(fp_reg, offset - reg_save_index * BytesPerWord, R1_SP);
343 } else {
344 assert(action == ACTION_COUNT_ONLY, "Sanity");
345 }
346 }
347 } else if (vm_reg->is_ConditionRegister()) {
348 // NOP. Conditions registers are covered by save_LR_CR
349 } else if (vm_reg->is_VectorRegister()) {
350 assert(SuperwordUseVSX, "or should not reach here");
351 VectorSRegister vs_reg = (vm_reg->as_VectorRegister()).to_vsr();
352 if (vs_reg->encoding() >= VSR32->encoding() && vs_reg->encoding() <= VSR51->encoding()) {
353 reg_save_index += (2 + (reg_save_index & 1)); // 2 slots + alignment if needed
354
355 Register spill_addr = R0;
356 int spill_offset = offset - reg_save_index * BytesPerWord;
357 if (action == ACTION_SAVE) {
358 _masm->stxv(vs_reg, spill_offset, R1_SP);
359 } else if (action == ACTION_RESTORE) {
360 _masm->lxv(vs_reg, spill_offset, R1_SP);
361 } else {
362 assert(action == ACTION_COUNT_ONLY, "Sanity");
363 }
364 }
365 } else {
366 if (vm_reg->is_SpecialRegister()) {
367 fatal("Special registers are unsupported. Found register %s", vm_reg->name());
368 } else {
369 fatal("Register type is not known");
370 }
371 }
372 }
373
374 return reg_save_index;
375 }
376
377 #endif // COMPILER2