1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2025, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "macroAssembler_ppc.hpp"
42 #include "nativeInst_ppc.hpp"
43 #include "runtime/javaThread.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "utilities/globalDefinitions.hpp"
46 #include "vm_version_ppc.hpp"
47 #ifdef COMPILER1
48 #include "c1/c1_LIRAssembler.hpp"
49 #include "c1/c1_MacroAssembler.hpp"
50 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
51 #endif
52 #ifdef COMPILER2
53 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
54 #endif
55
56 #define __ masm->
57
58 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
59 Register base, RegisterOrConstant ind_or_offs,
60 Register tmp1, Register tmp2, Register tmp3,
61 MacroAssembler::PreservationLevel preservation_level,
62 int extra_stack_space) {
63 if (ShenandoahSATBBarrier) {
64 __ block_comment("satb_barrier (shenandoahgc) {");
65 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level, extra_stack_space);
66 __ block_comment("} satb_barrier (shenandoahgc)");
67 }
68 }
69
70 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
71 Register base, RegisterOrConstant ind_or_offs,
72 Register dst,
73 Register tmp1, Register tmp2,
74 MacroAssembler::PreservationLevel preservation_level,
75 int extra_stack_space) {
76 if (ShenandoahLoadRefBarrier) {
77 __ block_comment("load_reference_barrier (shenandoahgc) {");
78 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level, extra_stack_space);
79 __ block_comment("} load_reference_barrier (shenandoahgc)");
80 }
81 }
82
83 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
84 Register src, Register dst, Register count,
85 Register preserve1, Register preserve2) {
86 Register R11_tmp = R11_scratch1;
87
88 assert_different_registers(src, dst, count, R11_tmp, noreg);
89 if (preserve1 != noreg) {
90 // Technically not required, but likely to indicate an error.
91 assert_different_registers(preserve1, preserve2);
92 }
93
94 /* ==== Check whether barrier is required (optimizations) ==== */
95 // Fast path: Component type of array is not a reference type.
96 if (!is_reference_type(type)) {
97 return;
98 }
99
100 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
101
102 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
103 // any useful information.
104 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahLoadRefBarrier) {
105 return;
106 }
107
108 __ block_comment("arraycopy_prologue (shenandoahgc) {");
109 Label skip_prologue;
110
111 // Fast path: Array is of length zero.
112 __ cmpdi(CR0, count, 0);
113 __ beq(CR0, skip_prologue);
114
115 /* ==== Check whether barrier is required (gc state) ==== */
116 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
117 R16_thread);
118
119 // The set of garbage collection states requiring barriers depends on the available barrier types and the
120 // type of the reference in question.
121 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
122 // for the garbage collector.
123 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
124 ? ShenandoahHeap::HAS_FORWARDED
125 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
126
127 __ andi_(R11_tmp, R11_tmp, required_states);
128 __ beq(CR0, skip_prologue);
129
130 /* ==== Invoke runtime ==== */
131 // Save to-be-preserved registers.
132 int highest_preserve_register_index = 0;
133 {
134 if (preserve1 != noreg && preserve1->is_volatile()) {
135 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
136 }
137 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
138 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
139 }
140
141 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
142 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
143 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
144
145 __ save_LR(R11_tmp);
146 __ push_frame_reg_args(-BytesPerWord * highest_preserve_register_index,
147 R11_tmp);
148 }
149
150 // Invoke runtime.
151 address jrt_address = nullptr;
152 if (UseCompressedOops) {
153 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
154 } else {
155 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
156 }
157 assert(jrt_address != nullptr, "jrt routine cannot be found");
158
159 __ call_VM_leaf(jrt_address, src, dst, count);
160
161 // Restore to-be-preserved registers.
162 {
163 __ pop_frame();
164 __ restore_LR(R11_tmp);
165
166 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
167 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
168 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
169
170 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
171 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
172 }
173 if (preserve1 != noreg && preserve1->is_volatile()) {
174 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
175 }
176 }
177
178 __ bind(skip_prologue);
179 __ block_comment("} arraycopy_prologue (shenandoahgc)");
180 }
181
182 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
183 Register dst, Register count,
184 Register preserve) {
185 if (ShenandoahCardBarrier && is_reference_type(type)) {
186 __ block_comment("arraycopy_epilogue (shenandoahgc) {");
187 gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
188 __ block_comment("} arraycopy_epilogue (shenandoahgc)");
189 }
190 }
191
192 // The to-be-enqueued value can either be determined
193 // - dynamically by passing the reference's address information (load mode) or
194 // - statically by passing a register the value is stored in (preloaded mode)
195 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
196 // - for incremental-update barriers.
197 //
198 // decorators: The previous value's decorator set.
199 // In "load mode", the value must equal '0'.
200 // base: Base register of the reference's address (load mode).
201 // In "preloaded mode", the register must equal 'noreg'.
202 // ind_or_offs: Index or offset of the reference's address (load mode).
203 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
204 // pre_val: Register holding the to-be-stored value (preloaded mode).
205 // In "load mode", this register acts as a temporary register and must
206 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
207 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
208 void ShenandoahBarrierSetAssembler::satb_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
209 Register base, RegisterOrConstant ind_or_offs,
210 Register pre_val,
211 Register tmp1, Register tmp2,
212 MacroAssembler::PreservationLevel preservation_level,
213 int extra_stack_space) {
214 assert(ShenandoahSATBBarrier, "Should be checked by caller");
215 assert_different_registers(tmp1, tmp2, pre_val, noreg);
216
217 Label skip_barrier;
218
219 /* ==== Determine necessary runtime invocation preservation measures ==== */
220 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
221 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
222 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
223
224 // Check whether marking is active.
225 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
226
227 __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
228 __ beq(CR0, skip_barrier);
229
230 /* ==== Determine the reference's previous value ==== */
231 bool preloaded_mode = base == noreg;
232 Register pre_val_save = noreg;
233
234 if (preloaded_mode) {
235 // Previous value has been passed to the method, so it must not be determined manually.
236 // In case 'pre_val' is a volatile register, it must be saved across the C-call
237 // as callers may depend on its value.
238 // Unless the general purposes registers are saved anyway, one of the temporary registers
239 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
240 if (!preserve_gp_registers && pre_val->is_volatile()) {
241 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
242 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
243 }
244
245 if ((decorators & IS_NOT_NULL) != 0) {
246 #ifdef ASSERT
247 __ cmpdi(CR0, pre_val, 0);
248 __ asm_assert_ne("null oop is not allowed");
249 #endif // ASSERT
250 } else {
251 __ cmpdi(CR0, pre_val, 0);
252 __ beq(CR0, skip_barrier);
253 }
254 } else {
255 // Load from the reference address to determine the reference's current value (before the store is being performed).
256 // Contrary to the given value in "preloaded mode", it is not necessary to preserve it.
257 assert(decorators == 0, "decorator set must be empty");
258 assert(base != noreg, "base must be a register");
259 assert(!ind_or_offs.is_register() || ind_or_offs.as_register() != noreg, "ind_or_offs must be a register");
260 if (UseCompressedOops) {
261 __ lwz(pre_val, ind_or_offs, base);
262 } else {
263 __ ld(pre_val, ind_or_offs, base);
264 }
265
266 __ cmpdi(CR0, pre_val, 0);
267 __ beq(CR0, skip_barrier);
268
269 if (UseCompressedOops) {
270 __ decode_heap_oop_not_null(pre_val);
271 }
272 }
273
274 /* ==== Try to enqueue the to-be-stored value directly into thread's local SATB mark queue ==== */
275 {
276 Label runtime;
277 Register Rbuffer = tmp1, Rindex = tmp2;
278
279 // Check whether the queue has enough capacity to store another oop.
280 // If not, jump to the runtime to commit the buffer and to allocate a new one.
281 // (The buffer's index corresponds to the amount of remaining free space.)
282 __ ld(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
283 __ cmpdi(CR0, Rindex, 0);
284 __ beq(CR0, runtime); // If index == 0 (buffer is full), goto runtime.
285
286 // Capacity suffices. Decrement the queue's size by the size of one oop.
287 // (The buffer is filled contrary to the heap's growing direction, i.e., it is filled downwards.)
288 __ addi(Rindex, Rindex, -wordSize);
289 __ std(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
290
291 // Enqueue the previous value and skip the invocation of the runtime.
292 __ ld(Rbuffer, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
293 __ stdx(pre_val, Rbuffer, Rindex);
294 __ b(skip_barrier);
295
296 __ bind(runtime);
297 }
298
299 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
300 // Save to-be-preserved registers.
301 int nbytes_save = 0;
302
303 if (needs_frame) {
304 if (preserve_gp_registers) {
305 nbytes_save = (preserve_fp_registers
306 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
307 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord + extra_stack_space;
308 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
309 }
310
311 __ save_LR(tmp1);
312 __ push_frame_reg_args(nbytes_save, tmp2);
313 }
314
315 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
316 assert(pre_val_save != noreg, "nv_save must not be noreg");
317
318 // 'pre_val' register must be saved manually unless general-purpose are preserved in general.
319 __ mr(pre_val_save, pre_val);
320 }
321
322 // Invoke runtime.
323 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
324
325 // Restore to-be-preserved registers.
326 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
327 __ mr(pre_val, pre_val_save);
328 }
329
330 if (needs_frame) {
331 __ pop_frame();
332 __ restore_LR(tmp1);
333
334 if (preserve_gp_registers) {
335 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
336 }
337 }
338
339 __ bind(skip_barrier);
340 }
341
342 // base: Base register of the reference's address.
343 // ind_or_offs: Index or offset of the reference's address (load mode).
344 // dst: Reference's address. In case the object has been evacuated, this is the to-space version
345 // of that object.
346 void ShenandoahBarrierSetAssembler::load_reference_barrier_impl(
347 MacroAssembler *masm, DecoratorSet decorators,
348 Register base, RegisterOrConstant ind_or_offs,
349 Register dst,
350 Register tmp1, Register tmp2,
351 MacroAssembler::PreservationLevel preservation_level,
352 int extra_stack_space) {
353 if (ind_or_offs.is_register()) {
354 assert_different_registers(tmp1, tmp2, base, ind_or_offs.as_register(), dst, noreg);
355 } else {
356 assert_different_registers(tmp1, tmp2, base, dst, noreg);
357 }
358
359 Label skip_barrier;
360
361 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
362 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
363 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
364 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
365 bool is_narrow = UseCompressedOops && !is_native;
366
367 /* ==== Check whether heap is stable ==== */
368 __ lbz(tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
369
370 if (is_strong) {
371 // For strong references, the heap is considered stable if "has forwarded" is not active.
372 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
373 __ beq(CR0, skip_barrier);
374 #ifdef ASSERT
375 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
376 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
377 __ asm_assert_ne("'has forwarded' is missing");
378 #endif // ASSERT
379 } else {
380 // For all non-strong references, the heap is considered stable if not any of "has forwarded",
381 // "root set processing", and "weak reference processing" is active.
382 // The additional phase conditions are in place to avoid the resurrection of weak references (see JDK-8266440).
383 Label skip_fastpath;
384 __ andi_(tmp1, tmp2, ShenandoahHeap::WEAK_ROOTS);
385 __ bne(CR0, skip_fastpath);
386
387 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
388 __ beq(CR0, skip_barrier);
389 #ifdef ASSERT
390 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
391 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
392 __ asm_assert_ne("'has forwarded' is missing");
393 #endif // ASSERT
394
395 __ bind(skip_fastpath);
396 }
397
398 /* ==== Check whether region is in collection set ==== */
399 if (is_strong) {
400 // Shenandoah stores metadata on regions in a continuous area of memory in which a single byte corresponds to
401 // an entire region of the shenandoah heap. At present, only the least significant bit is of significance
402 // and indicates whether the region is part of the collection set.
403 //
404 // All regions are of the same size and are always aligned by a power of two.
405 // Any address can thus be shifted by a fixed number of bits to retrieve the address prefix shared by
406 // all objects within that region (region identification bits).
407 //
408 // | unused bits | region identification bits | object identification bits |
409 // (Region size depends on a couple of criteria, such as page size, user-provided arguments and the max heap size.
410 // The number of object identification bits can thus not be determined at compile time.)
411 //
412 // ------------------------------------------------------- <--- cs (collection set) base address
413 // | lost space due to heap space base address -> 'ShenandoahHeap::in_cset_fast_test_addr()'
414 // | (region identification bits contain heap base offset)
415 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
416 // | collection set in the proper -> shift: 'region_size_bytes_shift_jint()'
417 // |
418 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
419 // + number of regions
420 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
421 __ srdi(tmp1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
422 __ lbzx(tmp2, tmp1, tmp2);
423 __ andi_(tmp2, tmp2, 1);
424 __ beq(CR0, skip_barrier);
425 }
426
427 /* ==== Invoke runtime ==== */
428 // Save to-be-preserved registers.
429 int nbytes_save = 0;
430
431 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
432 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
433 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
434
435 if (needs_frame) {
436 if (preserve_gp_registers) {
437 nbytes_save = (preserve_fp_registers
438 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
439 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord + extra_stack_space;
440 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
441 }
442
443 __ save_LR(tmp1);
444 __ push_frame_reg_args(nbytes_save, tmp1);
445 }
446
447 // Calculate the reference's absolute address.
448 __ add(R4_ARG2, ind_or_offs, base);
449
450 // Invoke runtime.
451 address jrt_address = nullptr;
452
453 if (is_strong) {
454 if (is_narrow) {
455 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
456 } else {
457 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
458 }
459 } else if (is_weak) {
460 if (is_narrow) {
461 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
462 } else {
463 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
464 }
465 } else {
466 assert(is_phantom, "only remaining strength");
467 assert(!is_narrow, "phantom access cannot be narrow");
468 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
469 }
470 assert(jrt_address != nullptr, "jrt routine cannot be found");
471
472 __ call_VM_leaf(jrt_address, dst /* reference */, R4_ARG2 /* reference address */);
473
474 // Restore to-be-preserved registers.
475 if (preserve_gp_registers) {
476 __ mr(R0, R3_RET);
477 } else {
478 __ mr_if_needed(dst, R3_RET);
479 }
480
481 if (needs_frame) {
482 __ pop_frame();
483 __ restore_LR(tmp1);
484
485 if (preserve_gp_registers) {
486 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
487 __ mr(dst, R0);
488 }
489 }
490
491 __ bind(skip_barrier);
492 }
493
494 // base: Base register of the reference's address.
495 // ind_or_offs: Index or offset of the reference's address.
496 // L_handle_null: An optional label that will be jumped to if the reference is null.
497 void ShenandoahBarrierSetAssembler::load_at(
498 MacroAssembler *masm, DecoratorSet decorators, BasicType type,
499 Register base, RegisterOrConstant ind_or_offs, Register dst,
500 Register tmp1, Register tmp2,
501 MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
502 // Register must not clash, except 'base' and 'dst'.
503 if (ind_or_offs.is_register()) {
504 if (base != noreg) {
505 assert_different_registers(tmp1, tmp2, base, ind_or_offs.register_or_noreg(), R0, noreg);
506 }
507 assert_different_registers(tmp1, tmp2, dst, ind_or_offs.register_or_noreg(), R0, noreg);
508 } else {
509 if (base == noreg) {
510 assert_different_registers(tmp1, tmp2, base, R0, noreg);
511 }
512 assert_different_registers(tmp1, tmp2, dst, R0, noreg);
513 }
514
515 /* ==== Apply load barrier, if required ==== */
516 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
517 assert(is_reference_type(type), "need_load_reference_barrier must check whether type is a reference type");
518
519 // If 'dst' clashes with either 'base' or 'ind_or_offs', use an intermediate result register
520 // to keep the values of those alive until the load reference barrier is applied.
521 Register intermediate_dst = (dst == base || (ind_or_offs.is_register() && dst == ind_or_offs.as_register()))
522 ? tmp2
523 : dst;
524
525 BarrierSetAssembler::load_at(masm, decorators, type,
526 base, ind_or_offs,
527 intermediate_dst,
528 tmp1, noreg,
529 preservation_level, L_handle_null);
530
531 load_reference_barrier(masm, decorators,
532 base, ind_or_offs,
533 intermediate_dst,
534 tmp1, R0,
535 preservation_level);
536
537 __ mr_if_needed(dst, intermediate_dst);
538 } else {
539 BarrierSetAssembler::load_at(masm, decorators, type,
540 base, ind_or_offs,
541 dst,
542 tmp1, tmp2,
543 preservation_level, L_handle_null);
544 }
545
546 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
547 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
548 if (ShenandoahSATBBarrier) {
549 __ block_comment("keep_alive_barrier (shenandoahgc) {");
550 satb_barrier_impl(masm, 0, noreg, noreg, dst, tmp1, tmp2, preservation_level);
551 __ block_comment("} keep_alive_barrier (shenandoahgc)");
552 }
553 }
554 }
555
556 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
557 assert(ShenandoahCardBarrier, "Should have been checked by caller");
558 assert_different_registers(base, tmp, R0);
559
560 if (ind_or_offs.is_constant()) {
561 __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
562 } else {
563 __ add(base, ind_or_offs.as_register(), base);
564 }
565
566 __ ld(tmp, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread); /* tmp = *[R16_thread + card_table_offset] */
567 __ srdi(base, base, CardTable::card_shift());
568 __ li(R0, CardTable::dirty_card_val());
569 __ stbx(R0, tmp, base);
570 }
571
572 // base: Base register of the reference's address.
573 // ind_or_offs: Index or offset of the reference's address.
574 // val: To-be-stored value/reference's new value.
575 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
576 Register base, RegisterOrConstant ind_or_offs, Register val,
577 Register tmp1, Register tmp2, Register tmp3,
578 MacroAssembler::PreservationLevel preservation_level) {
579 // 1: non-reference types require no barriers
580 if (!is_reference_type(type)) {
581 BarrierSetAssembler::store_at(masm, decorators, type,
582 base, ind_or_offs,
583 val,
584 tmp1, tmp2, tmp3,
585 preservation_level);
586 return;
587 }
588
589 bool storing_non_null = (val != noreg);
590
591 // 2: pre-barrier: SATB needs the previous value
592 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
593 satb_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
594 }
595
596 // Store!
597 BarrierSetAssembler::store_at(masm, decorators, type,
598 base, ind_or_offs,
599 val,
600 tmp1, tmp2, tmp3,
601 preservation_level);
602
603 // 3: post-barrier: card barrier needs store address
604 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
605 card_barrier(masm, base, ind_or_offs, tmp1);
606 }
607 }
608
609 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
610 Register dst, Register jni_env, Register obj,
611 Register tmp, Label &slowpath) {
612 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
613
614 assert_different_registers(jni_env, obj, tmp);
615
616 Label done;
617
618 // Fast path: Reference is null (JNI tags are zero for null pointers).
619 __ cmpdi(CR0, obj, 0);
620 __ beq(CR0, done);
621
622 // Resolve jobject using standard implementation.
623 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
624
625 // Check whether heap is stable.
626 __ lbz(tmp,
627 in_bytes(ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()),
628 jni_env);
629
630 __ andi_(tmp, tmp, ShenandoahHeap::EVACUATION | ShenandoahHeap::HAS_FORWARDED);
631 __ bne(CR0, slowpath);
632
633 __ bind(done);
634 __ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
635 }
636
637 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
638 Register obj, Register tmp, Label &slow_path) {
639 __ block_comment("try_peek_weak_handle_in_nmethod (shenandoahgc) {");
640
641 assert_different_registers(weak_handle, tmp, noreg);
642 assert_different_registers(obj, tmp, noreg);
643
644
645 Label done;
646
647 // Peek weak handle using the standard implementation.
648 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
649
650 // Check if the reference is null, and if it is, take the fast path.
651 __ cmpdi(CR0, obj, 0);
652 __ beq(CR0, done);
653
654 // Check if the heap is under weak-reference/roots processing, in
655 // which case we need to take the slow path.
656 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
657 __ andi_(tmp, tmp, ShenandoahHeap::WEAK_ROOTS);
658 __ bne(CR0, slow_path);
659 __ bind(done);
660
661 __ block_comment("} try_peek_weak_handle_in_nmethod (shenandoahgc)");
662 }
663
664 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
665 Register addr, Register count, Register preserve) {
666 assert(ShenandoahCardBarrier, "Should have been checked by caller");
667 assert_different_registers(addr, count, R0);
668
669 Label L_skip_loop, L_store_loop;
670
671 __ sldi_(count, count, LogBytesPerHeapOop);
672
673 // Zero length? Skip.
674 __ beq(CR0, L_skip_loop);
675
676 __ addi(count, count, -BytesPerHeapOop);
677 __ add(count, addr, count);
678 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
679 __ srdi(addr, addr, CardTable::card_shift());
680 __ srdi(count, count, CardTable::card_shift());
681 __ subf(count, addr, count);
682 __ ld(R0, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
683 __ add(addr, addr, R0);
684 __ addi(count, count, 1);
685 __ li(R0, 0);
686 __ mtctr(count);
687
688 // Byte store loop
689 __ bind(L_store_loop);
690 __ stb(R0, 0, addr);
691 __ addi(addr, addr, 1);
692 __ bdnz(L_store_loop);
693 __ bind(L_skip_loop);
694 }
695
696 #undef __
697
698 #ifdef COMPILER1
699
700 #define __ ce->masm()->
701
702 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
703 __ block_comment("keepalive_barrier_stub (shenandoahgc) {");
704 __ bind(*stub->entry());
705
706 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
707
708 Register obj = stub->obj()->as_register();
709
710 // If 'do_load()' returns false, the to-be-stored value is already available in 'obj'
711 if (stub->do_load()) {
712 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false);
713 }
714
715 // Fast path: reference is null.
716 __ cmpdi(CR0, obj, 0);
717 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
718
719 // Argument passing via the stack.
720 __ std(obj, -8, R1_SP);
721
722 address blob_addr = bs->keepalive_barrier_stub();
723 __ load_const_optimized(R0, blob_addr);
724 __ call_stub(R0);
725
726 __ b(*stub->continuation());
727 __ block_comment("} keepalive_barrier_stub (shenandoahgc)");
728 }
729
730 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
731 __ block_comment("load_reference_barrier_stub (shenandoahgc) {");
732
733 __ bind(*stub->entry());
734
735 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
736
737 Register obj = stub->obj()->as_register();
738 Register addr = stub->addr()->as_pointer_register();
739 Register slow_result = stub->slow_result()->as_register();
740 assert_different_registers(obj, addr, slow_result);
741 assert(slow_result == R3_RET, "C1 must know about our slow call result register");
742
743 // Argument passing via the stack.
744 __ std(obj, -8, R1_SP);
745 __ std(addr, -16, R1_SP);
746
747 address blob_addr = bs->load_reference_barrier_stub(stub->decorators());
748 __ load_const_optimized(R0, blob_addr);
749 __ call_stub(R0);
750 if (obj != slow_result) {
751 __ mr(obj, slow_result);
752 }
753
754 __ b(*stub->continuation());
755 __ block_comment("} load_reference_barrier_stub (shenandoahgc)");
756 }
757
758 #undef __
759
760 #define __ sasm->
761
762 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_runtime_stub(StubAssembler* sasm) {
763 __ block_comment("keepalive_barrier_runtime_stub (shenandoahgc) {");
764
765 Register obj = R3_ARG1;
766 Register tmp1 = R11_scratch1;
767 Register tmp2 = R12_scratch2;
768
769 // Save registers we are about to clobber
770 __ std(obj, -16, R1_SP);
771 __ std(tmp1, -24, R1_SP);
772 __ std(tmp2, -32, R1_SP);
773
774 // Pull the arguments from stack
775 __ ld(obj, -8, R1_SP);
776
777 satb_barrier(sasm, noreg, noreg, obj, tmp1, tmp2, MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS, 4 * BytesPerWord);
778
779 // Restore registers
780 __ ld(tmp2, -32, R1_SP);
781 __ ld(tmp1, -24, R1_SP);
782 __ ld(obj, -16, R1_SP);
783
784 __ blr();
785 __ block_comment("} keepalive_barrier_runtime_stub (shenandoahgc)");
786 }
787
788 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
789 __ block_comment("load_reference_barrier_runtime_stub (shenandoahgc) {");
790
791 Register obj = R3_ARG1;
792 Register addr = R4_ARG2;
793 Register tmp1 = R11_scratch1;
794 Register tmp2 = R12_scratch2;
795
796 // Save registers we are about to clobber
797 __ std(addr, -24, R1_SP);
798 __ std(tmp1, -32, R1_SP);
799 __ std(tmp2, -40, R1_SP);
800
801 // Pull the arguments from the stack
802 __ ld(obj, -8, R1_SP);
803 __ ld(addr, -16, R1_SP);
804
805 load_reference_barrier(sasm, decorators, addr, noreg, obj, tmp1, tmp2,
806 MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS, 5 * BytesPerWord);
807
808 // Restore registers
809 __ ld(tmp2, -40, R1_SP);
810 __ ld(tmp1, -32, R1_SP);
811 __ ld(addr, -24, R1_SP);
812
813 __ blr();
814 __ block_comment("} load_reference_barrier_runtime_stub (shenandoahgc)");
815 }
816
817 #undef __
818
819 #endif // COMPILER1
820
821 #ifdef COMPILER2
822
823 #undef __
824 #define __ masm->
825
826 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Register addr, int disp, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
827 if (is_narrow) {
828 __ lwz(dst, disp, addr);
829 } else {
830 __ ld(dst, disp, addr);
831 }
832 if (is_acquire) {
833 __ twi_0(dst);
834 __ isync();
835 }
836
837 ShenandoahBarrierStubC2::load_post(masm, node, dst, Address(addr, disp), tmp1, tmp2, is_narrow);
838 }
839
840 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
841 Register dst, int disp, bool dst_narrow, Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
842
843 ShenandoahBarrierStubC2::store_pre(masm, node, Address(dst, disp), tmp1, tmp2, tmp3, dst_narrow);
844
845 if (dst_narrow && !src_narrow) {
846 // Need to encode into tmp, because we cannot clobber src.
847 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
848 src = __ encode_heap_oop(tmp1, src);
849 } else {
850 src = __ encode_heap_oop_not_null(tmp1, src);
851 }
852 }
853 if (dst_narrow) {
854 __ stw(src, disp, dst);
855 } else {
856 __ std(src, disp, dst);
857 }
858
859 ShenandoahBarrierStubC2::store_post(masm, node, Address(dst, disp), tmp1, tmp2);
860 }
861
862 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
863 Register oldval, Register newval, Register tmp1, Register tmp2, bool exchange, bool narrow, bool weak, bool acquire) {
864
865 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, res, tmp1, tmp2, narrow);
866
867 Register dest_current = exchange ? res : R0;
868 Label no_update;
869 int semantics = MacroAssembler::MemBarNone;
870
871 if (acquire) {
872 semantics = support_IRIW_for_not_multiple_copy_atomic_cpu ?
873 MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter;
874 }
875
876 if (!exchange) { __ li(res, 0); }
877 if (narrow) {
878 __ cmpxchgw(CR0, dest_current, oldval, newval, addr,
879 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
880 noreg, &no_update, true, weak);
881 } else {
882 __ cmpxchgd(CR0, dest_current, oldval, newval, addr,
883 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
884 noreg, &no_update, true, weak);
885 }
886 if (!exchange) { __ li(res, 1); }
887
888 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
889
890 __ bind(no_update);
891 }
892
893 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval, Register newval, Register addr, Register tmp1, Register tmp2) {
894 bool is_narrow = node->bottom_type()->isa_narrowoop();
895
896 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, preval, tmp1, tmp2, is_narrow);
897
898 if (is_narrow) {
899 __ getandsetw(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
900 } else {
901 __ getandsetd(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
902 }
903
904 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
905 __ isync();
906 } else {
907 __ sync();
908 }
909
910 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
911 }
912
913 #undef __
914 #define __ masm.
915
916 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
917 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
918 assert_different_registers(tmp1, tmp2, address.index(), address.base());
919
920 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
921 if (address.index() == noreg) {
922 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
923 } else {
924 __ add(tmp2, address.index(), address.base());
925 if (address.disp() != 0) {
926 __ addi(tmp2, tmp2, address.disp());
927 }
928 }
929 __ srdi(tmp2, tmp2, CardTable::card_shift());
930 __ li(R0, CardTable::dirty_card_val());
931 __ stbx(R0, tmp2, tmp1);
932 }
933
934 void ShenandoahBarrierStubC2::patchable_jump_if_gc_state(MacroAssembler& masm, const char test_state, Label* L_target) {
935 // Emit the unconditional branch in the first version of the method.
936 // Let the rest of runtime figure out how to manage it.
937 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(test_state, false)));
938 __ b(*L_target);
939 }
940
941 void ShenandoahBarrierStubC2::patchable_jump_if_not_gc_state(MacroAssembler& masm, const char test_state, Label* L_target) {
942 // Emit the unconditional branch in the first version of the method.
943 // Let the rest of runtime figure out how to manage it.
944 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(test_state, true)));
945 __ b(*L_target);
946 }
947
948 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
949 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
950 patchable_jump_if_gc_state(masm, test_state, entry());
951 __ bind(*continuation());
952 }
953
954 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
955 NativeInstruction* ni = nativeInstruction_at(pc);
956 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
957 NativeGeneralJump* jmp = nativeGeneralJump_at(pc);
958 return jmp->jump_destination();
959 }
960
961 static void check_at(bool cond, address pc, const char* msg) {
962 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x",
963 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3));
964 }
965
966 static bool is_patchable_nop(address pc) {
967 if (*(pc + 0) != 0x00) return false;
968 if (*(pc + 1) != 0x00) return false;
969 if (*(pc + 2) != 0x00) return false;
970 if (*(pc + 3) != 0x60) return false;
971 return true;
972 }
973
974 static void insert_patchable_nop(address pc) {
975 *reinterpret_cast<int32_t*>(pc) = 0x60000000;
976 }
977
978 bool ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
979 NativeInstruction* ni = nativeInstruction_at(pc);
980 bool patching = ni->is_jump();
981 if (patching) {
982 insert_patchable_nop(pc);
983 }
984 check_at(is_patchable_nop(pc), pc, "Should be nop");
985 return patching;
986 }
987
988 bool ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
989 bool patching = is_patchable_nop(pc);
990 if (patching) {
991 NativeGeneralJump::insert_unconditional(pc, stub_addr);
992 }
993
994 NativeInstruction* ni = nativeInstruction_at(pc);
995 check_at(ni->is_jump(), pc, "Should be jump");
996 check_at(nativeGeneralJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
997 return patching;
998 }
999
1000 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1001 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1002 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1003
1004 __ bind(*entry());
1005
1006 // If we need to load ourselves, do it here.
1007 if (_do_load) {
1008 if (_narrow) {
1009 __ lwz(_obj, _addr.disp(), _addr.base());
1010 } else {
1011 __ ld(_obj, _addr.disp(), _addr.base());
1012 }
1013 }
1014
1015 // If the object is null, there is no point in applying barriers.
1016 maybe_far_jump_if_zero(masm, _obj);
1017
1018 // We need to make sure that loads done by callers survive across slow-path calls.
1019 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1020 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
1021 if (!_do_load || needs_both_barriers) {
1022 preserve(_obj);
1023 }
1024
1025 // Go for barriers. Barriers can return straight to continuation, as long
1026 // as another barrier is not needed and we can reach the fastpath.
1027 if (needs_both_barriers) {
1028 keepalive(masm, nullptr);
1029 lrb(masm);
1030 } else if (_needs_keep_alive_barrier) {
1031 keepalive(masm, continuation());
1032 } else if (_needs_load_ref_barrier) {
1033 lrb(masm);
1034 } else {
1035 ShouldNotReachHere();
1036 }
1037 }
1038
1039 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1040 __ cmpdi(CR0, reg, 0);
1041 // Branch to continuation if equal
1042 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
1043 }
1044
1045 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1046 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1047 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1048 Label L_through, L_slowpath;
1049
1050 // If another barrier is enabled as well, do a check for a specific barrier.
1051 if (_needs_load_ref_barrier) {
1052 assert(L_done == nullptr, "Should be");
1053 char state_to_check = ShenandoahHeap::MARKING;
1054 patchable_jump_if_not_gc_state(masm, state_to_check, &L_through);
1055 }
1056
1057 // Fast-path: put object into buffer.
1058 // If buffer is already full, go slow.
1059 __ ld(_tmp1, index_offset, R16_thread);
1060 __ cmpdi(CR0, _tmp1, 0);
1061 __ beq(CR0, L_slowpath);
1062 __ addi(_tmp1, _tmp1, -wordSize);
1063 __ std(_tmp1, index_offset, R16_thread);
1064 __ ld(_tmp2, buffer_offset, R16_thread);
1065
1066 // Store the object in queue.
1067 // If object is narrow, we need to decode it before inserting.
1068 if (_narrow) {
1069 __ add(_tmp2, _tmp2, _tmp1);
1070 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1071 __ stdx(decoded, _tmp2);
1072 } else {
1073 __ stdx(_obj, _tmp2, _tmp1);
1074 }
1075
1076 // Fast-path exits here.
1077 if (L_done != nullptr) {
1078 __ b(*L_done);
1079 } else {
1080 __ b(L_through);
1081 }
1082
1083 // Slow-path: call runtime to handle.
1084 __ bind(L_slowpath);
1085
1086 {
1087 SaveLiveRegisters slr(&masm, this);
1088
1089 // Go to runtime and handle the rest there.
1090 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1091 }
1092
1093 if (L_done != nullptr) {
1094 __ b(*L_done);
1095 } else {
1096 __ bind(L_through);
1097 }
1098 }
1099
1100 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1101 Label L_slow;
1102
1103 // If another barrier is enabled as well, do a check for a specific barrier.
1104 if (_needs_keep_alive_barrier) {
1105 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1106 patchable_jump_if_not_gc_state(masm, state_to_check, continuation());
1107 }
1108
1109 // If weak references are being processed, weak/phantom loads need to go slow,
1110 // regardless of their cset status.
1111 if (_needs_load_ref_weak_barrier) {
1112 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
1113 patchable_jump_if_gc_state(masm, state_to_check, &L_slow);
1114 }
1115
1116 // Cset-check. Fall-through to slow if in collection set.
1117 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1118 if (_narrow) {
1119 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1120 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1121 } else {
1122 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1123 }
1124 __ lbzx(_tmp2, _tmp2, _tmp1);
1125 maybe_far_jump_if_zero(masm, _tmp2);
1126
1127 // Slow path
1128 __ bind(L_slow);
1129
1130 // Obj is the result, need to temporarily stop preserving it.
1131 bool is_obj_preserved = is_preserved(_obj);
1132 if (is_obj_preserved) {
1133 dont_preserve(_obj);
1134 }
1135 {
1136 SaveLiveRegisters slr(&masm, this);
1137
1138 // Shuffle in the arguments. The end result should be:
1139 // c_rarg0 <-- obj
1140 // c_rarg1 <-- lea(addr)
1141 Register c_rarg0 = R3_ARG1;
1142 Register c_rarg1 = R4_ARG2;
1143 if (c_rarg0 == _obj) {
1144 __ addi(c_rarg1, _addr.base(), _addr.disp());
1145 } else if (c_rarg1 == _obj) {
1146 __ mr(_tmp1, c_rarg1);
1147 __ addi(c_rarg1, _addr.base(), _addr.disp());
1148 __ mr(c_rarg0, _tmp1);
1149 } else {
1150 assert_different_registers(c_rarg1, _obj);
1151 __ addi(c_rarg1, _addr.base(), _addr.disp());
1152 __ mr(c_rarg0, _obj);
1153 }
1154
1155 // Go to runtime and handle the rest there.
1156 __ call_VM_leaf(lrb_runtime_entry_addr(), c_rarg0, c_rarg1);
1157
1158 // Save the result where needed.
1159 if (_obj != R3_RET) {
1160 __ mr(_obj, R3_RET);
1161 }
1162 }
1163 if (is_obj_preserved) {
1164 preserve(_obj);
1165 }
1166
1167 __ b(*continuation());
1168 }
1169
1170 int ShenandoahBarrierStubC2::available_gp_registers() {
1171 Unimplemented(); // Not used
1172 return 0;
1173 }
1174
1175 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1176 Unimplemented(); // Not used
1177 return true;
1178 }
1179
1180 void ShenandoahBarrierStubC2::post_init() {
1181 // Do nothing.
1182 }
1183
1184 #endif // COMPILER2