1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2025, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahRuntime.hpp"
38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
39 #include "interpreter/interpreter.hpp"
40 #include "macroAssembler_ppc.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "utilities/globalDefinitions.hpp"
44 #include "vm_version_ppc.hpp"
45 #ifdef COMPILER1
46 #include "c1/c1_LIRAssembler.hpp"
47 #include "c1/c1_MacroAssembler.hpp"
48 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
49 #endif
50 #ifdef COMPILER2
51 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
52 #endif
53
54 #define __ masm->
55
56 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
57 Register base, RegisterOrConstant ind_or_offs,
58 Register tmp1, Register tmp2, Register tmp3,
59 MacroAssembler::PreservationLevel preservation_level,
60 int extra_stack_space) {
61 if (ShenandoahSATBBarrier) {
62 __ block_comment("satb_barrier (shenandoahgc) {");
63 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level, extra_stack_space);
64 __ block_comment("} satb_barrier (shenandoahgc)");
65 }
66 }
67
68 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
69 Register base, RegisterOrConstant ind_or_offs,
70 Register dst,
71 Register tmp1, Register tmp2,
72 MacroAssembler::PreservationLevel preservation_level,
73 int extra_stack_space) {
74 if (ShenandoahLoadRefBarrier) {
75 __ block_comment("load_reference_barrier (shenandoahgc) {");
76 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level, extra_stack_space);
77 __ block_comment("} load_reference_barrier (shenandoahgc)");
78 }
79 }
80
81 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
82 Register src, Register dst, Register count,
83 Register preserve1, Register preserve2) {
84 Register R11_tmp = R11_scratch1;
85
86 assert_different_registers(src, dst, count, R11_tmp, noreg);
87 if (preserve1 != noreg) {
88 // Technically not required, but likely to indicate an error.
89 assert_different_registers(preserve1, preserve2);
90 }
91
92 /* ==== Check whether barrier is required (optimizations) ==== */
93 // Fast path: Component type of array is not a reference type.
94 if (!is_reference_type(type)) {
95 return;
96 }
97
98 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
99
100 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
101 // any useful information.
102 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahLoadRefBarrier) {
103 return;
104 }
105
106 __ block_comment("arraycopy_prologue (shenandoahgc) {");
107 Label skip_prologue;
108
109 // Fast path: Array is of length zero.
110 __ cmpdi(CR0, count, 0);
111 __ beq(CR0, skip_prologue);
112
113 /* ==== Check whether barrier is required (gc state) ==== */
114 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
115 R16_thread);
116
117 // The set of garbage collection states requiring barriers depends on the available barrier types and the
118 // type of the reference in question.
119 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
120 // for the garbage collector.
121 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
122 ? ShenandoahHeap::HAS_FORWARDED
123 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
124
125 __ andi_(R11_tmp, R11_tmp, required_states);
126 __ beq(CR0, skip_prologue);
127
128 /* ==== Invoke runtime ==== */
129 // Save to-be-preserved registers.
130 int highest_preserve_register_index = 0;
131 {
132 if (preserve1 != noreg && preserve1->is_volatile()) {
133 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
134 }
135 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
136 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
137 }
138
139 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
140 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
141 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
142
143 __ save_LR(R11_tmp);
144 __ push_frame_reg_args(-BytesPerWord * highest_preserve_register_index,
145 R11_tmp);
146 }
147
148 // Invoke runtime.
149 address jrt_address = nullptr;
150 if (UseCompressedOops) {
151 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
152 } else {
153 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
154 }
155 assert(jrt_address != nullptr, "jrt routine cannot be found");
156
157 __ call_VM_leaf(jrt_address, src, dst, count);
158
159 // Restore to-be-preserved registers.
160 {
161 __ pop_frame();
162 __ restore_LR(R11_tmp);
163
164 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
165 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
166 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
167
168 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
169 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
170 }
171 if (preserve1 != noreg && preserve1->is_volatile()) {
172 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
173 }
174 }
175
176 __ bind(skip_prologue);
177 __ block_comment("} arraycopy_prologue (shenandoahgc)");
178 }
179
180 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
181 Register dst, Register count,
182 Register preserve) {
183 if (ShenandoahCardBarrier && is_reference_type(type)) {
184 __ block_comment("arraycopy_epilogue (shenandoahgc) {");
185 gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
186 __ block_comment("} arraycopy_epilogue (shenandoahgc)");
187 }
188 }
189
190 // The to-be-enqueued value can either be determined
191 // - dynamically by passing the reference's address information (load mode) or
192 // - statically by passing a register the value is stored in (preloaded mode)
193 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
194 // - for incremental-update barriers.
195 //
196 // decorators: The previous value's decorator set.
197 // In "load mode", the value must equal '0'.
198 // base: Base register of the reference's address (load mode).
199 // In "preloaded mode", the register must equal 'noreg'.
200 // ind_or_offs: Index or offset of the reference's address (load mode).
201 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
202 // pre_val: Register holding the to-be-stored value (preloaded mode).
203 // In "load mode", this register acts as a temporary register and must
204 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
205 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
206 void ShenandoahBarrierSetAssembler::satb_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
207 Register base, RegisterOrConstant ind_or_offs,
208 Register pre_val,
209 Register tmp1, Register tmp2,
210 MacroAssembler::PreservationLevel preservation_level,
211 int extra_stack_space) {
212 assert(ShenandoahSATBBarrier, "Should be checked by caller");
213 assert_different_registers(tmp1, tmp2, pre_val, noreg);
214
215 Label skip_barrier;
216
217 /* ==== Determine necessary runtime invocation preservation measures ==== */
218 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
219 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
220 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
221
222 // Check whether marking is active.
223 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
224
225 __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
226 __ beq(CR0, skip_barrier);
227
228 /* ==== Determine the reference's previous value ==== */
229 bool preloaded_mode = base == noreg;
230 Register pre_val_save = noreg;
231
232 if (preloaded_mode) {
233 // Previous value has been passed to the method, so it must not be determined manually.
234 // In case 'pre_val' is a volatile register, it must be saved across the C-call
235 // as callers may depend on its value.
236 // Unless the general purposes registers are saved anyway, one of the temporary registers
237 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
238 if (!preserve_gp_registers && pre_val->is_volatile()) {
239 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
240 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
241 }
242
243 if ((decorators & IS_NOT_NULL) != 0) {
244 #ifdef ASSERT
245 __ cmpdi(CR0, pre_val, 0);
246 __ asm_assert_ne("null oop is not allowed");
247 #endif // ASSERT
248 } else {
249 __ cmpdi(CR0, pre_val, 0);
250 __ beq(CR0, skip_barrier);
251 }
252 } else {
253 // Load from the reference address to determine the reference's current value (before the store is being performed).
254 // Contrary to the given value in "preloaded mode", it is not necessary to preserve it.
255 assert(decorators == 0, "decorator set must be empty");
256 assert(base != noreg, "base must be a register");
257 assert(!ind_or_offs.is_register() || ind_or_offs.as_register() != noreg, "ind_or_offs must be a register");
258 if (UseCompressedOops) {
259 __ lwz(pre_val, ind_or_offs, base);
260 } else {
261 __ ld(pre_val, ind_or_offs, base);
262 }
263
264 __ cmpdi(CR0, pre_val, 0);
265 __ beq(CR0, skip_barrier);
266
267 if (UseCompressedOops) {
268 __ decode_heap_oop_not_null(pre_val);
269 }
270 }
271
272 /* ==== Try to enqueue the to-be-stored value directly into thread's local SATB mark queue ==== */
273 {
274 Label runtime;
275 Register Rbuffer = tmp1, Rindex = tmp2;
276
277 // Check whether the queue has enough capacity to store another oop.
278 // If not, jump to the runtime to commit the buffer and to allocate a new one.
279 // (The buffer's index corresponds to the amount of remaining free space.)
280 __ ld(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
281 __ cmpdi(CR0, Rindex, 0);
282 __ beq(CR0, runtime); // If index == 0 (buffer is full), goto runtime.
283
284 // Capacity suffices. Decrement the queue's size by the size of one oop.
285 // (The buffer is filled contrary to the heap's growing direction, i.e., it is filled downwards.)
286 __ addi(Rindex, Rindex, -wordSize);
287 __ std(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
288
289 // Enqueue the previous value and skip the invocation of the runtime.
290 __ ld(Rbuffer, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
291 __ stdx(pre_val, Rbuffer, Rindex);
292 __ b(skip_barrier);
293
294 __ bind(runtime);
295 }
296
297 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
298 // Save to-be-preserved registers.
299 int nbytes_save = 0;
300
301 if (needs_frame) {
302 if (preserve_gp_registers) {
303 nbytes_save = (preserve_fp_registers
304 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
305 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord + extra_stack_space;
306 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
307 }
308
309 __ save_LR(tmp1);
310 __ push_frame_reg_args(nbytes_save, tmp2);
311 }
312
313 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
314 assert(pre_val_save != noreg, "nv_save must not be noreg");
315
316 // 'pre_val' register must be saved manually unless general-purpose are preserved in general.
317 __ mr(pre_val_save, pre_val);
318 }
319
320 // Invoke runtime.
321 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
322
323 // Restore to-be-preserved registers.
324 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
325 __ mr(pre_val, pre_val_save);
326 }
327
328 if (needs_frame) {
329 __ pop_frame();
330 __ restore_LR(tmp1);
331
332 if (preserve_gp_registers) {
333 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
334 }
335 }
336
337 __ bind(skip_barrier);
338 }
339
340 // base: Base register of the reference's address.
341 // ind_or_offs: Index or offset of the reference's address (load mode).
342 // dst: Reference's address. In case the object has been evacuated, this is the to-space version
343 // of that object.
344 void ShenandoahBarrierSetAssembler::load_reference_barrier_impl(
345 MacroAssembler *masm, DecoratorSet decorators,
346 Register base, RegisterOrConstant ind_or_offs,
347 Register dst,
348 Register tmp1, Register tmp2,
349 MacroAssembler::PreservationLevel preservation_level,
350 int extra_stack_space) {
351 if (ind_or_offs.is_register()) {
352 assert_different_registers(tmp1, tmp2, base, ind_or_offs.as_register(), dst, noreg);
353 } else {
354 assert_different_registers(tmp1, tmp2, base, dst, noreg);
355 }
356
357 Label skip_barrier;
358
359 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
360 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
361 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
362 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
363 bool is_narrow = UseCompressedOops && !is_native;
364
365 /* ==== Check whether heap is stable ==== */
366 __ lbz(tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
367
368 if (is_strong) {
369 // For strong references, the heap is considered stable if "has forwarded" is not active.
370 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
371 __ beq(CR0, skip_barrier);
372 #ifdef ASSERT
373 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
374 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
375 __ asm_assert_ne("'has forwarded' is missing");
376 #endif // ASSERT
377 } else {
378 // For all non-strong references, the heap is considered stable if not any of "has forwarded",
379 // "root set processing", and "weak reference processing" is active.
380 // The additional phase conditions are in place to avoid the resurrection of weak references (see JDK-8266440).
381 Label skip_fastpath;
382 __ andi_(tmp1, tmp2, ShenandoahHeap::WEAK_ROOTS);
383 __ bne(CR0, skip_fastpath);
384
385 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
386 __ beq(CR0, skip_barrier);
387 #ifdef ASSERT
388 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
389 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
390 __ asm_assert_ne("'has forwarded' is missing");
391 #endif // ASSERT
392
393 __ bind(skip_fastpath);
394 }
395
396 /* ==== Check whether region is in collection set ==== */
397 if (is_strong) {
398 // Shenandoah stores metadata on regions in a continuous area of memory in which a single byte corresponds to
399 // an entire region of the shenandoah heap. At present, only the least significant bit is of significance
400 // and indicates whether the region is part of the collection set.
401 //
402 // All regions are of the same size and are always aligned by a power of two.
403 // Any address can thus be shifted by a fixed number of bits to retrieve the address prefix shared by
404 // all objects within that region (region identification bits).
405 //
406 // | unused bits | region identification bits | object identification bits |
407 // (Region size depends on a couple of criteria, such as page size, user-provided arguments and the max heap size.
408 // The number of object identification bits can thus not be determined at compile time.)
409 //
410 // ------------------------------------------------------- <--- cs (collection set) base address
411 // | lost space due to heap space base address -> 'ShenandoahHeap::in_cset_fast_test_addr()'
412 // | (region identification bits contain heap base offset)
413 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
414 // | collection set in the proper -> shift: 'region_size_bytes_shift_jint()'
415 // |
416 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
417 // + number of regions
418 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
419 __ srdi(tmp1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
420 __ lbzx(tmp2, tmp1, tmp2);
421 __ andi_(tmp2, tmp2, 1);
422 __ beq(CR0, skip_barrier);
423 }
424
425 /* ==== Invoke runtime ==== */
426 // Save to-be-preserved registers.
427 int nbytes_save = 0;
428
429 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
430 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
431 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
432
433 if (needs_frame) {
434 if (preserve_gp_registers) {
435 nbytes_save = (preserve_fp_registers
436 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
437 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord + extra_stack_space;
438 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
439 }
440
441 __ save_LR(tmp1);
442 __ push_frame_reg_args(nbytes_save, tmp1);
443 }
444
445 // Calculate the reference's absolute address.
446 __ add(R4_ARG2, ind_or_offs, base);
447
448 // Invoke runtime.
449 address jrt_address = nullptr;
450
451 if (is_strong) {
452 if (is_narrow) {
453 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
454 } else {
455 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
456 }
457 } else if (is_weak) {
458 if (is_narrow) {
459 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
460 } else {
461 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
462 }
463 } else {
464 assert(is_phantom, "only remaining strength");
465 assert(!is_narrow, "phantom access cannot be narrow");
466 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
467 }
468 assert(jrt_address != nullptr, "jrt routine cannot be found");
469
470 __ call_VM_leaf(jrt_address, dst /* reference */, R4_ARG2 /* reference address */);
471
472 // Restore to-be-preserved registers.
473 if (preserve_gp_registers) {
474 __ mr(R0, R3_RET);
475 } else {
476 __ mr_if_needed(dst, R3_RET);
477 }
478
479 if (needs_frame) {
480 __ pop_frame();
481 __ restore_LR(tmp1);
482
483 if (preserve_gp_registers) {
484 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
485 __ mr(dst, R0);
486 }
487 }
488
489 __ bind(skip_barrier);
490 }
491
492 // base: Base register of the reference's address.
493 // ind_or_offs: Index or offset of the reference's address.
494 // L_handle_null: An optional label that will be jumped to if the reference is null.
495 void ShenandoahBarrierSetAssembler::load_at(
496 MacroAssembler *masm, DecoratorSet decorators, BasicType type,
497 Register base, RegisterOrConstant ind_or_offs, Register dst,
498 Register tmp1, Register tmp2,
499 MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
500 // Register must not clash, except 'base' and 'dst'.
501 if (ind_or_offs.is_register()) {
502 if (base != noreg) {
503 assert_different_registers(tmp1, tmp2, base, ind_or_offs.register_or_noreg(), R0, noreg);
504 }
505 assert_different_registers(tmp1, tmp2, dst, ind_or_offs.register_or_noreg(), R0, noreg);
506 } else {
507 if (base == noreg) {
508 assert_different_registers(tmp1, tmp2, base, R0, noreg);
509 }
510 assert_different_registers(tmp1, tmp2, dst, R0, noreg);
511 }
512
513 /* ==== Apply load barrier, if required ==== */
514 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
515 assert(is_reference_type(type), "need_load_reference_barrier must check whether type is a reference type");
516
517 // If 'dst' clashes with either 'base' or 'ind_or_offs', use an intermediate result register
518 // to keep the values of those alive until the load reference barrier is applied.
519 Register intermediate_dst = (dst == base || (ind_or_offs.is_register() && dst == ind_or_offs.as_register()))
520 ? tmp2
521 : dst;
522
523 BarrierSetAssembler::load_at(masm, decorators, type,
524 base, ind_or_offs,
525 intermediate_dst,
526 tmp1, noreg,
527 preservation_level, L_handle_null);
528
529 load_reference_barrier(masm, decorators,
530 base, ind_or_offs,
531 intermediate_dst,
532 tmp1, R0,
533 preservation_level);
534
535 __ mr_if_needed(dst, intermediate_dst);
536 } else {
537 BarrierSetAssembler::load_at(masm, decorators, type,
538 base, ind_or_offs,
539 dst,
540 tmp1, tmp2,
541 preservation_level, L_handle_null);
542 }
543
544 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
545 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
546 if (ShenandoahSATBBarrier) {
547 __ block_comment("keep_alive_barrier (shenandoahgc) {");
548 satb_barrier_impl(masm, 0, noreg, noreg, dst, tmp1, tmp2, preservation_level);
549 __ block_comment("} keep_alive_barrier (shenandoahgc)");
550 }
551 }
552 }
553
554 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
555 assert(ShenandoahCardBarrier, "Should have been checked by caller");
556 assert_different_registers(base, tmp, R0);
557
558 if (ind_or_offs.is_constant()) {
559 __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
560 } else {
561 __ add(base, ind_or_offs.as_register(), base);
562 }
563
564 __ ld(tmp, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread); /* tmp = *[R16_thread + card_table_offset] */
565 __ srdi(base, base, CardTable::card_shift());
566 __ li(R0, CardTable::dirty_card_val());
567 __ stbx(R0, tmp, base);
568 }
569
570 // base: Base register of the reference's address.
571 // ind_or_offs: Index or offset of the reference's address.
572 // val: To-be-stored value/reference's new value.
573 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
574 Register base, RegisterOrConstant ind_or_offs, Register val,
575 Register tmp1, Register tmp2, Register tmp3,
576 MacroAssembler::PreservationLevel preservation_level) {
577 // 1: non-reference types require no barriers
578 if (!is_reference_type(type)) {
579 BarrierSetAssembler::store_at(masm, decorators, type,
580 base, ind_or_offs,
581 val,
582 tmp1, tmp2, tmp3,
583 preservation_level);
584 return;
585 }
586
587 bool storing_non_null = (val != noreg);
588
589 // 2: pre-barrier: SATB needs the previous value
590 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
591 satb_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
592 }
593
594 // Store!
595 BarrierSetAssembler::store_at(masm, decorators, type,
596 base, ind_or_offs,
597 val,
598 tmp1, tmp2, tmp3,
599 preservation_level);
600
601 // 3: post-barrier: card barrier needs store address
602 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
603 card_barrier(masm, base, ind_or_offs, tmp1);
604 }
605 }
606
607 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
608 Register dst, Register jni_env, Register obj,
609 Register tmp, Label &slowpath) {
610 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
611
612 assert_different_registers(jni_env, obj, tmp);
613
614 Label done;
615
616 // Fast path: Reference is null (JNI tags are zero for null pointers).
617 __ cmpdi(CR0, obj, 0);
618 __ beq(CR0, done);
619
620 // Resolve jobject using standard implementation.
621 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
622
623 // Check whether heap is stable.
624 __ lbz(tmp,
625 in_bytes(ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()),
626 jni_env);
627
628 __ andi_(tmp, tmp, ShenandoahHeap::EVACUATION | ShenandoahHeap::HAS_FORWARDED);
629 __ bne(CR0, slowpath);
630
631 __ bind(done);
632 __ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
633 }
634
635 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
636 Register obj, Register tmp, Label &slow_path) {
637 __ block_comment("try_peek_weak_handle_in_nmethod (shenandoahgc) {");
638
639 assert_different_registers(weak_handle, tmp, noreg);
640 assert_different_registers(obj, tmp, noreg);
641
642
643 Label done;
644
645 // Peek weak handle using the standard implementation.
646 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
647
648 // Check if the reference is null, and if it is, take the fast path.
649 __ cmpdi(CR0, obj, 0);
650 __ beq(CR0, done);
651
652 // Check if the heap is under weak-reference/roots processing, in
653 // which case we need to take the slow path.
654 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
655 __ andi_(tmp, tmp, ShenandoahHeap::WEAK_ROOTS);
656 __ bne(CR0, slow_path);
657 __ bind(done);
658
659 __ block_comment("} try_peek_weak_handle_in_nmethod (shenandoahgc)");
660 }
661
662 void ShenandoahBarrierSetAssembler::check_oop(MacroAssembler *masm, Register obj, const char* msg) {
663 if (!VerifyOops) {
664 return;
665 }
666
667 __ mr(R0, obj);
668
669 // This routine is sometimes called before applying GC barriers.
670 // With +COH, verification can touch the klass that may end up loading forwarding pointer instead.
671 Label L_skip;
672 if (UseCompactObjectHeaders) {
673 __ lbz(R0, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
674 __ andi_(R0, R0, ShenandoahHeap::HAS_FORWARDED);
675 __ bne(CR0, L_skip);
676 }
677
678 __ verify_oop(R0, msg);
679 __ bind(L_skip);
680 }
681
682 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
683 Register addr, Register count, Register preserve) {
684 assert(ShenandoahCardBarrier, "Should have been checked by caller");
685 assert_different_registers(addr, count, R0);
686
687 Label L_skip_loop, L_store_loop;
688
689 __ sldi_(count, count, LogBytesPerHeapOop);
690
691 // Zero length? Skip.
692 __ beq(CR0, L_skip_loop);
693
694 __ addi(count, count, -BytesPerHeapOop);
695 __ add(count, addr, count);
696 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
697 __ srdi(addr, addr, CardTable::card_shift());
698 __ srdi(count, count, CardTable::card_shift());
699 __ subf(count, addr, count);
700 __ ld(R0, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
701 __ add(addr, addr, R0);
702 __ addi(count, count, 1);
703 __ li(R0, 0);
704 __ mtctr(count);
705
706 // Byte store loop
707 __ bind(L_store_loop);
708 __ stb(R0, 0, addr);
709 __ addi(addr, addr, 1);
710 __ bdnz(L_store_loop);
711 __ bind(L_skip_loop);
712 }
713
714 #undef __
715
716 #ifdef COMPILER1
717
718 #define __ ce->masm()->
719
720 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
721 __ block_comment("keepalive_barrier_stub (shenandoahgc) {");
722 __ bind(*stub->entry());
723
724 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
725
726 Register obj = stub->obj()->as_register();
727
728 // If 'do_load()' returns false, the to-be-stored value is already available in 'obj'
729 if (stub->do_load()) {
730 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false);
731 }
732
733 // Fast path: reference is null.
734 __ cmpdi(CR0, obj, 0);
735 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
736
737 // Argument passing via the stack.
738 __ std(obj, -8, R1_SP);
739
740 address blob_addr = bs->keepalive_barrier_stub();
741 __ load_const_optimized(R0, blob_addr);
742 __ call_stub(R0);
743
744 __ b(*stub->continuation());
745 __ block_comment("} keepalive_barrier_stub (shenandoahgc)");
746 }
747
748 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
749 __ block_comment("load_reference_barrier_stub (shenandoahgc) {");
750
751 __ bind(*stub->entry());
752
753 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
754
755 Register obj = stub->obj()->as_register();
756 Register addr = stub->addr()->as_pointer_register();
757 Register slow_result = stub->slow_result()->as_register();
758 assert_different_registers(obj, addr, slow_result);
759 assert(slow_result == R3_RET, "C1 must know about our slow call result register");
760
761 // Argument passing via the stack.
762 __ std(obj, -8, R1_SP);
763 __ std(addr, -16, R1_SP);
764
765 address blob_addr = bs->load_reference_barrier_stub(stub->decorators());
766 __ load_const_optimized(R0, blob_addr);
767 __ call_stub(R0);
768 if (obj != slow_result) {
769 __ mr(obj, slow_result);
770 }
771
772 __ b(*stub->continuation());
773 __ block_comment("} load_reference_barrier_stub (shenandoahgc)");
774 }
775
776 #undef __
777
778 #define __ sasm->
779
780 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_runtime_stub(StubAssembler* sasm) {
781 __ block_comment("keepalive_barrier_runtime_stub (shenandoahgc) {");
782
783 Register obj = R3_ARG1;
784 Register tmp1 = R11_scratch1;
785 Register tmp2 = R12_scratch2;
786
787 // Save registers we are about to clobber
788 __ std(obj, -16, R1_SP);
789 __ std(tmp1, -24, R1_SP);
790 __ std(tmp2, -32, R1_SP);
791
792 // Pull the arguments from stack
793 __ ld(obj, -8, R1_SP);
794
795 satb_barrier(sasm, noreg, noreg, obj, tmp1, tmp2, MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS, 4 * BytesPerWord);
796
797 // Restore registers
798 __ ld(tmp2, -32, R1_SP);
799 __ ld(tmp1, -24, R1_SP);
800 __ ld(obj, -16, R1_SP);
801
802 __ blr();
803 __ block_comment("} keepalive_barrier_runtime_stub (shenandoahgc)");
804 }
805
806 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
807 __ block_comment("load_reference_barrier_runtime_stub (shenandoahgc) {");
808
809 Register obj = R3_ARG1;
810 Register addr = R4_ARG2;
811 Register tmp1 = R11_scratch1;
812 Register tmp2 = R12_scratch2;
813
814 // Save registers we are about to clobber
815 __ std(addr, -24, R1_SP);
816 __ std(tmp1, -32, R1_SP);
817 __ std(tmp2, -40, R1_SP);
818
819 // Pull the arguments from the stack
820 __ ld(obj, -8, R1_SP);
821 __ ld(addr, -16, R1_SP);
822
823 load_reference_barrier(sasm, decorators, addr, noreg, obj, tmp1, tmp2,
824 MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS, 5 * BytesPerWord);
825
826 // Restore registers
827 __ ld(tmp2, -40, R1_SP);
828 __ ld(tmp1, -32, R1_SP);
829 __ ld(addr, -24, R1_SP);
830
831 __ blr();
832 __ block_comment("} load_reference_barrier_runtime_stub (shenandoahgc)");
833 }
834
835 #undef __
836
837 #endif // COMPILER1
838
839 #ifdef COMPILER2
840
841 #undef __
842 #define __ masm->
843
844 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Register addr, int disp, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
845 if (is_narrow) {
846 __ lwz(dst, disp, addr);
847 } else {
848 __ ld(dst, disp, addr);
849 }
850 if (is_acquire) {
851 __ twi_0(dst);
852 __ isync();
853 }
854
855 ShenandoahBarrierStubC2::load_post(masm, node, dst, Address(addr, disp), tmp1, tmp2, is_narrow);
856 }
857
858 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
859 Register dst, int disp, bool dst_narrow, Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
860
861 ShenandoahBarrierStubC2::store_pre(masm, node, Address(dst, disp), tmp1, tmp2, tmp3, dst_narrow);
862
863 if (dst_narrow && !src_narrow) {
864 // Need to encode into tmp, because we cannot clobber src.
865 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
866 src = __ encode_heap_oop(tmp1, src);
867 } else {
868 src = __ encode_heap_oop_not_null(tmp1, src);
869 }
870 }
871 if (dst_narrow) {
872 __ stw(src, disp, dst);
873 } else {
874 __ std(src, disp, dst);
875 }
876
877 ShenandoahBarrierStubC2::store_post(masm, node, Address(dst, disp), tmp1, tmp2);
878 }
879
880 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
881 Register oldval, Register newval, Register tmp1, Register tmp2, bool exchange, bool narrow, bool weak, bool acquire) {
882
883 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, res, tmp1, tmp2, narrow);
884
885 Register dest_current = exchange ? res : R0;
886 Label no_update;
887 int semantics = MacroAssembler::MemBarNone;
888
889 if (acquire) {
890 semantics = support_IRIW_for_not_multiple_copy_atomic_cpu ?
891 MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter;
892 }
893
894 if (!exchange) { __ li(res, 0); }
895 if (narrow) {
896 __ cmpxchgw(CR0, dest_current, oldval, newval, addr,
897 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
898 noreg, &no_update, true, weak);
899 } else {
900 __ cmpxchgd(CR0, dest_current, oldval, newval, addr,
901 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
902 noreg, &no_update, true, weak);
903 }
904 if (!exchange) { __ li(res, 1); }
905
906 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
907
908 __ bind(no_update);
909 }
910
911 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval, Register newval, Register addr, Register tmp1, Register tmp2) {
912 bool is_narrow = node->bottom_type()->isa_narrowoop();
913
914 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, preval, tmp1, tmp2, is_narrow);
915
916 if (is_narrow) {
917 __ getandsetw(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
918 } else {
919 __ getandsetd(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
920 }
921
922 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
923 __ isync();
924 } else {
925 __ sync();
926 }
927
928 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
929 }
930
931 #undef __
932 #define __ masm.
933
934 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
935 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
936 assert_different_registers(tmp1, tmp2, address.index(), address.base());
937
938 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
939 if (address.index() == noreg) {
940 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
941 } else {
942 __ add(tmp2, address.index(), address.base());
943 if (address.disp() != 0) {
944 __ addi(tmp2, tmp2, address.disp());
945 }
946 }
947 __ srdi(tmp2, tmp2, CardTable::card_shift());
948 __ li(R0, CardTable::dirty_card_val());
949 __ stbx(R0, tmp2, tmp1);
950 }
951
952 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
953 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
954
955 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)), R16_thread);
956 __ cmpdi(CR0, tmp, 0);
957 // Branch to entry if not equal
958 __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *entry());
959 // This is were the slowpath stub will return to
960 __ bind(*continuation());
961 }
962
963 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
964 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
965 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
966
967 __ bind(*entry());
968
969 // If we need to load ourselves, do it here.
970 if (_do_load) {
971 if (_narrow) {
972 __ lwz(_obj, _addr.disp(), _addr.base());
973 } else {
974 __ ld(_obj, _addr.disp(), _addr.base());
975 }
976 }
977
978 // If the object is null, there is no point in applying barriers.
979 maybe_far_jump_if_zero(masm, _obj);
980
981 // We need to make sure that loads done by callers survive across slow-path calls.
982 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
983 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
984 if (!_do_load || needs_both_barriers) {
985 preserve(_obj);
986 }
987
988 // Go for barriers. Barriers can return straight to continuation, as long
989 // as another barrier is not needed and we can reach the fastpath.
990 if (needs_both_barriers) {
991 keepalive(masm, nullptr);
992 lrb(masm);
993 } else if (_needs_keep_alive_barrier) {
994 keepalive(masm, continuation());
995 } else if (_needs_load_ref_barrier) {
996 lrb(masm);
997 } else {
998 ShouldNotReachHere();
999 }
1000 }
1001
1002 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1003 __ cmpdi(CR0, reg, 0);
1004 // Branch to continuation if equal
1005 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
1006 }
1007
1008 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1009 const int gcstate_offset = in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING));
1010 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1011 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1012 Label L_through, L_slowpath;
1013
1014 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1015 if (_needs_load_ref_barrier) {
1016 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
1017 __ lbz(_tmp1, gcstate_offset, R16_thread);
1018 __ cmpdi(CR0, _tmp1, 0);
1019 __ beq(CR0, L_through);
1020 }
1021
1022 // Fast-path: put object into buffer.
1023 // If buffer is already full, go slow.
1024 __ ld(_tmp1, index_offset, R16_thread);
1025 __ cmpdi(CR0, _tmp1, 0);
1026 __ beq(CR0, L_slowpath);
1027 __ addi(_tmp1, _tmp1, -wordSize);
1028 __ std(_tmp1, index_offset, R16_thread);
1029 __ ld(_tmp2, buffer_offset, R16_thread);
1030
1031 // Store the object in queue.
1032 // If object is narrow, we need to decode it before inserting.
1033 if (_narrow) {
1034 __ add(_tmp2, _tmp2, _tmp1);
1035 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1036 __ stdx(decoded, _tmp2);
1037 } else {
1038 __ stdx(_obj, _tmp2, _tmp1);
1039 }
1040
1041 // Fast-path exits here.
1042 if (L_done != nullptr) {
1043 __ b(*L_done);
1044 } else {
1045 __ b(L_through);
1046 }
1047
1048 // Slow-path: call runtime to handle.
1049 __ bind(L_slowpath);
1050
1051 {
1052 SaveLiveRegisters slr(&masm, this);
1053
1054 // Go to runtime and handle the rest there.
1055 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1056 }
1057
1058 if (L_done != nullptr) {
1059 __ b(*L_done);
1060 } else {
1061 __ bind(L_through);
1062 }
1063 }
1064
1065 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1066 Label L_slow;
1067
1068 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1069 if (_needs_keep_alive_barrier) {
1070 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1071 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)), R16_thread);
1072 maybe_far_jump_if_zero(masm, _tmp1);
1073 }
1074
1075 // If weak references are being processed, weak/phantom loads need to go slow,
1076 // regardless of their cset status.
1077 if (_needs_load_ref_weak_barrier) {
1078 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)), R16_thread);
1079 __ cmpdi(CR0, _tmp1, 0);
1080 __ bne(CR0, L_slow);
1081 }
1082
1083 // Cset-check. Fall-through to slow if in collection set.
1084 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1085 if (_narrow) {
1086 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1087 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1088 } else {
1089 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1090 }
1091 __ lbzx(_tmp2, _tmp2, _tmp1);
1092 maybe_far_jump_if_zero(masm, _tmp2);
1093
1094 // Slow path
1095 __ bind(L_slow);
1096
1097 // Obj is the result, need to temporarily stop preserving it.
1098 bool is_obj_preserved = is_preserved(_obj);
1099 if (is_obj_preserved) {
1100 dont_preserve(_obj);
1101 }
1102 {
1103 SaveLiveRegisters slr(&masm, this);
1104
1105 // Shuffle in the arguments. The end result should be:
1106 // c_rarg0 <-- obj
1107 // c_rarg1 <-- lea(addr)
1108 Register c_rarg0 = R3_ARG1;
1109 Register c_rarg1 = R4_ARG2;
1110 if (c_rarg0 == _obj) {
1111 __ addi(c_rarg1, _addr.base(), _addr.disp());
1112 } else if (c_rarg1 == _obj) {
1113 __ mr(_tmp1, c_rarg1);
1114 __ addi(c_rarg1, _addr.base(), _addr.disp());
1115 __ mr(c_rarg0, _tmp1);
1116 } else {
1117 assert_different_registers(c_rarg1, _obj);
1118 __ addi(c_rarg1, _addr.base(), _addr.disp());
1119 __ mr(c_rarg0, _obj);
1120 }
1121
1122 // Go to runtime and handle the rest there.
1123 __ call_VM_leaf(lrb_runtime_entry_addr(), c_rarg0, c_rarg1);
1124
1125 // Save the result where needed.
1126 if (_obj != R3_RET) {
1127 __ mr(_obj, R3_RET);
1128 }
1129 }
1130 if (is_obj_preserved) {
1131 preserve(_obj);
1132 }
1133
1134 __ b(*continuation());
1135 }
1136
1137 int ShenandoahBarrierStubC2::available_gp_registers() {
1138 Unimplemented(); // Not used
1139 return 0;
1140 }
1141
1142 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1143 Unimplemented(); // Not used
1144 return true;
1145 }
1146
1147 void ShenandoahBarrierStubC2::post_init() {
1148 // Do nothing.
1149 }
1150
1151 #endif // COMPILER2