1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2025, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "macroAssembler_ppc.hpp"
42 #include "nativeInst_ppc.hpp"
43 #include "runtime/javaThread.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "utilities/globalDefinitions.hpp"
46 #include "vm_version_ppc.hpp"
47 #ifdef COMPILER1
48 #include "c1/c1_LIRAssembler.hpp"
49 #include "c1/c1_MacroAssembler.hpp"
50 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
51 #endif
52 #ifdef COMPILER2
53 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
54 #include "opto/output.hpp"
55 #endif
56
57 #define __ masm->
58
59 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
60 Register base, RegisterOrConstant ind_or_offs,
61 Register tmp1, Register tmp2, Register tmp3,
62 MacroAssembler::PreservationLevel preservation_level,
63 int extra_stack_space) {
64 if (ShenandoahSATBBarrier) {
65 __ block_comment("satb_barrier (shenandoahgc) {");
66 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level, extra_stack_space);
67 __ block_comment("} satb_barrier (shenandoahgc)");
68 }
69 }
70
71 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
72 Register base, RegisterOrConstant ind_or_offs,
73 Register dst,
74 Register tmp1, Register tmp2,
75 MacroAssembler::PreservationLevel preservation_level,
76 int extra_stack_space) {
77 if (ShenandoahLoadRefBarrier) {
78 __ block_comment("load_reference_barrier (shenandoahgc) {");
79 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level, extra_stack_space);
80 __ block_comment("} load_reference_barrier (shenandoahgc)");
81 }
82 }
83
84 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
85 Register src, Register dst, Register count,
86 Register preserve1, Register preserve2) {
87 Register R11_tmp = R11_scratch1;
88
89 assert_different_registers(src, dst, count, R11_tmp, noreg);
90 if (preserve1 != noreg) {
91 // Technically not required, but likely to indicate an error.
92 assert_different_registers(preserve1, preserve2);
93 }
94
95 /* ==== Check whether barrier is required (optimizations) ==== */
96 // Fast path: Component type of array is not a reference type.
97 if (!is_reference_type(type)) {
98 return;
99 }
100
101 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
102
103 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
104 // any useful information.
105 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahLoadRefBarrier) {
106 return;
107 }
108
109 __ block_comment("arraycopy_prologue (shenandoahgc) {");
110 Label skip_prologue;
111
112 // Fast path: Array is of length zero.
113 __ cmpdi(CR0, count, 0);
114 __ beq(CR0, skip_prologue);
115
116 /* ==== Check whether barrier is required (gc state) ==== */
117 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
118 R16_thread);
119
120 // The set of garbage collection states requiring barriers depends on the available barrier types and the
121 // type of the reference in question.
122 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
123 // for the garbage collector.
124 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
125 ? ShenandoahHeap::HAS_FORWARDED
126 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
127
128 __ andi_(R11_tmp, R11_tmp, required_states);
129 __ beq(CR0, skip_prologue);
130
131 /* ==== Invoke runtime ==== */
132 // Save to-be-preserved registers.
133 int highest_preserve_register_index = 0;
134 {
135 if (preserve1 != noreg && preserve1->is_volatile()) {
136 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
137 }
138 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
139 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
140 }
141
142 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
143 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
144 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
145
146 __ save_LR(R11_tmp);
147 __ push_frame_reg_args(-BytesPerWord * highest_preserve_register_index,
148 R11_tmp);
149 }
150
151 // Invoke runtime.
152 address jrt_address = nullptr;
153 if (UseCompressedOops) {
154 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
155 } else {
156 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
157 }
158 assert(jrt_address != nullptr, "jrt routine cannot be found");
159
160 __ call_VM_leaf(jrt_address, src, dst, count);
161
162 // Restore to-be-preserved registers.
163 {
164 __ pop_frame();
165 __ restore_LR(R11_tmp);
166
167 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
168 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
169 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
170
171 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
172 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
173 }
174 if (preserve1 != noreg && preserve1->is_volatile()) {
175 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
176 }
177 }
178
179 __ bind(skip_prologue);
180 __ block_comment("} arraycopy_prologue (shenandoahgc)");
181 }
182
183 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
184 Register dst, Register count,
185 Register preserve) {
186 if (ShenandoahCardBarrier && is_reference_type(type)) {
187 __ block_comment("arraycopy_epilogue (shenandoahgc) {");
188 gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
189 __ block_comment("} arraycopy_epilogue (shenandoahgc)");
190 }
191 }
192
193 // The to-be-enqueued value can either be determined
194 // - dynamically by passing the reference's address information (load mode) or
195 // - statically by passing a register the value is stored in (preloaded mode)
196 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
197 // - for incremental-update barriers.
198 //
199 // decorators: The previous value's decorator set.
200 // In "load mode", the value must equal '0'.
201 // base: Base register of the reference's address (load mode).
202 // In "preloaded mode", the register must equal 'noreg'.
203 // ind_or_offs: Index or offset of the reference's address (load mode).
204 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
205 // pre_val: Register holding the to-be-stored value (preloaded mode).
206 // In "load mode", this register acts as a temporary register and must
207 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
208 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
209 void ShenandoahBarrierSetAssembler::satb_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
210 Register base, RegisterOrConstant ind_or_offs,
211 Register pre_val,
212 Register tmp1, Register tmp2,
213 MacroAssembler::PreservationLevel preservation_level,
214 int extra_stack_space) {
215 assert(ShenandoahSATBBarrier, "Should be checked by caller");
216 assert_different_registers(tmp1, tmp2, pre_val, noreg);
217
218 Label skip_barrier;
219
220 /* ==== Determine necessary runtime invocation preservation measures ==== */
221 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
222 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
223 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
224
225 // Check whether marking is active.
226 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
227
228 __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
229 __ beq(CR0, skip_barrier);
230
231 /* ==== Determine the reference's previous value ==== */
232 bool preloaded_mode = base == noreg;
233 Register pre_val_save = noreg;
234
235 if (preloaded_mode) {
236 // Previous value has been passed to the method, so it must not be determined manually.
237 // In case 'pre_val' is a volatile register, it must be saved across the C-call
238 // as callers may depend on its value.
239 // Unless the general purposes registers are saved anyway, one of the temporary registers
240 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
241 if (!preserve_gp_registers && pre_val->is_volatile()) {
242 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
243 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
244 }
245
246 if ((decorators & IS_NOT_NULL) != 0) {
247 #ifdef ASSERT
248 __ cmpdi(CR0, pre_val, 0);
249 __ asm_assert_ne("null oop is not allowed");
250 #endif // ASSERT
251 } else {
252 __ cmpdi(CR0, pre_val, 0);
253 __ beq(CR0, skip_barrier);
254 }
255 } else {
256 // Load from the reference address to determine the reference's current value (before the store is being performed).
257 // Contrary to the given value in "preloaded mode", it is not necessary to preserve it.
258 assert(decorators == 0, "decorator set must be empty");
259 assert(base != noreg, "base must be a register");
260 assert(!ind_or_offs.is_register() || ind_or_offs.as_register() != noreg, "ind_or_offs must be a register");
261 if (UseCompressedOops) {
262 __ lwz(pre_val, ind_or_offs, base);
263 } else {
264 __ ld(pre_val, ind_or_offs, base);
265 }
266
267 __ cmpdi(CR0, pre_val, 0);
268 __ beq(CR0, skip_barrier);
269
270 if (UseCompressedOops) {
271 __ decode_heap_oop_not_null(pre_val);
272 }
273 }
274
275 /* ==== Try to enqueue the to-be-stored value directly into thread's local SATB mark queue ==== */
276 {
277 Label runtime;
278 Register Rbuffer = tmp1, Rindex = tmp2;
279
280 // Check whether the queue has enough capacity to store another oop.
281 // If not, jump to the runtime to commit the buffer and to allocate a new one.
282 // (The buffer's index corresponds to the amount of remaining free space.)
283 __ ld(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
284 __ cmpdi(CR0, Rindex, 0);
285 __ beq(CR0, runtime); // If index == 0 (buffer is full), goto runtime.
286
287 // Capacity suffices. Decrement the queue's size by the size of one oop.
288 // (The buffer is filled contrary to the heap's growing direction, i.e., it is filled downwards.)
289 __ addi(Rindex, Rindex, -wordSize);
290 __ std(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
291
292 // Enqueue the previous value and skip the invocation of the runtime.
293 __ ld(Rbuffer, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
294 __ stdx(pre_val, Rbuffer, Rindex);
295 __ b(skip_barrier);
296
297 __ bind(runtime);
298 }
299
300 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
301 // Save to-be-preserved registers.
302 int nbytes_save = 0;
303
304 if (needs_frame) {
305 if (preserve_gp_registers) {
306 nbytes_save = (preserve_fp_registers
307 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
308 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord + extra_stack_space;
309 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
310 }
311
312 __ save_LR(tmp1);
313 __ push_frame_reg_args(nbytes_save, tmp2);
314 }
315
316 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
317 assert(pre_val_save != noreg, "nv_save must not be noreg");
318
319 // 'pre_val' register must be saved manually unless general-purpose are preserved in general.
320 __ mr(pre_val_save, pre_val);
321 }
322
323 // Invoke runtime.
324 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
325
326 // Restore to-be-preserved registers.
327 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
328 __ mr(pre_val, pre_val_save);
329 }
330
331 if (needs_frame) {
332 __ pop_frame();
333 __ restore_LR(tmp1);
334
335 if (preserve_gp_registers) {
336 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
337 }
338 }
339
340 __ bind(skip_barrier);
341 }
342
343 // base: Base register of the reference's address.
344 // ind_or_offs: Index or offset of the reference's address (load mode).
345 // dst: Reference's address. In case the object has been evacuated, this is the to-space version
346 // of that object.
347 void ShenandoahBarrierSetAssembler::load_reference_barrier_impl(
348 MacroAssembler *masm, DecoratorSet decorators,
349 Register base, RegisterOrConstant ind_or_offs,
350 Register dst,
351 Register tmp1, Register tmp2,
352 MacroAssembler::PreservationLevel preservation_level,
353 int extra_stack_space) {
354 if (ind_or_offs.is_register()) {
355 assert_different_registers(tmp1, tmp2, base, ind_or_offs.as_register(), dst, noreg);
356 } else {
357 assert_different_registers(tmp1, tmp2, base, dst, noreg);
358 }
359
360 Label skip_barrier;
361
362 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
363 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
364 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
365 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
366 bool is_narrow = UseCompressedOops && !is_native;
367
368 /* ==== Check whether heap is stable ==== */
369 __ lbz(tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
370
371 if (is_strong) {
372 // For strong references, the heap is considered stable if "has forwarded" is not active.
373 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
374 __ beq(CR0, skip_barrier);
375 #ifdef ASSERT
376 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
377 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
378 __ asm_assert_ne("'has forwarded' is missing");
379 #endif // ASSERT
380 } else {
381 // For all non-strong references, the heap is considered stable if not any of "has forwarded",
382 // "root set processing", and "weak reference processing" is active.
383 // The additional phase conditions are in place to avoid the resurrection of weak references (see JDK-8266440).
384 Label skip_fastpath;
385 __ andi_(tmp1, tmp2, ShenandoahHeap::WEAK_ROOTS);
386 __ bne(CR0, skip_fastpath);
387
388 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
389 __ beq(CR0, skip_barrier);
390 #ifdef ASSERT
391 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
392 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
393 __ asm_assert_ne("'has forwarded' is missing");
394 #endif // ASSERT
395
396 __ bind(skip_fastpath);
397 }
398
399 /* ==== Check whether region is in collection set ==== */
400 if (is_strong) {
401 // Shenandoah stores metadata on regions in a continuous area of memory in which a single byte corresponds to
402 // an entire region of the shenandoah heap. At present, only the least significant bit is of significance
403 // and indicates whether the region is part of the collection set.
404 //
405 // All regions are of the same size and are always aligned by a power of two.
406 // Any address can thus be shifted by a fixed number of bits to retrieve the address prefix shared by
407 // all objects within that region (region identification bits).
408 //
409 // | unused bits | region identification bits | object identification bits |
410 // (Region size depends on a couple of criteria, such as page size, user-provided arguments and the max heap size.
411 // The number of object identification bits can thus not be determined at compile time.)
412 //
413 // ------------------------------------------------------- <--- cs (collection set) base address
414 // | lost space due to heap space base address -> 'ShenandoahHeap::in_cset_fast_test_addr()'
415 // | (region identification bits contain heap base offset)
416 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
417 // | collection set in the proper -> shift: 'region_size_bytes_shift_jint()'
418 // |
419 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
420 // + number of regions
421 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
422 __ srdi(tmp1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
423 __ lbzx(tmp2, tmp1, tmp2);
424 __ andi_(tmp2, tmp2, 1);
425 __ beq(CR0, skip_barrier);
426 }
427
428 /* ==== Invoke runtime ==== */
429 // Save to-be-preserved registers.
430 int nbytes_save = 0;
431
432 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
433 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
434 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
435
436 if (needs_frame) {
437 if (preserve_gp_registers) {
438 nbytes_save = (preserve_fp_registers
439 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
440 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord + extra_stack_space;
441 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
442 }
443
444 __ save_LR(tmp1);
445 __ push_frame_reg_args(nbytes_save, tmp1);
446 }
447
448 // Calculate the reference's absolute address.
449 __ add(R4_ARG2, ind_or_offs, base);
450
451 // Invoke runtime.
452 address jrt_address = nullptr;
453
454 if (is_strong) {
455 if (is_narrow) {
456 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
457 } else {
458 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
459 }
460 } else if (is_weak) {
461 if (is_narrow) {
462 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
463 } else {
464 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
465 }
466 } else {
467 assert(is_phantom, "only remaining strength");
468 assert(!is_narrow, "phantom access cannot be narrow");
469 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
470 }
471 assert(jrt_address != nullptr, "jrt routine cannot be found");
472
473 __ call_VM_leaf(jrt_address, dst /* reference */, R4_ARG2 /* reference address */);
474
475 // Restore to-be-preserved registers.
476 if (preserve_gp_registers) {
477 __ mr(R0, R3_RET);
478 } else {
479 __ mr_if_needed(dst, R3_RET);
480 }
481
482 if (needs_frame) {
483 __ pop_frame();
484 __ restore_LR(tmp1);
485
486 if (preserve_gp_registers) {
487 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
488 __ mr(dst, R0);
489 }
490 }
491
492 __ bind(skip_barrier);
493 }
494
495 // base: Base register of the reference's address.
496 // ind_or_offs: Index or offset of the reference's address.
497 // L_handle_null: An optional label that will be jumped to if the reference is null.
498 void ShenandoahBarrierSetAssembler::load_at(
499 MacroAssembler *masm, DecoratorSet decorators, BasicType type,
500 Register base, RegisterOrConstant ind_or_offs, Register dst,
501 Register tmp1, Register tmp2,
502 MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
503 // Register must not clash, except 'base' and 'dst'.
504 if (ind_or_offs.is_register()) {
505 if (base != noreg) {
506 assert_different_registers(tmp1, tmp2, base, ind_or_offs.register_or_noreg(), R0, noreg);
507 }
508 assert_different_registers(tmp1, tmp2, dst, ind_or_offs.register_or_noreg(), R0, noreg);
509 } else {
510 if (base == noreg) {
511 assert_different_registers(tmp1, tmp2, base, R0, noreg);
512 }
513 assert_different_registers(tmp1, tmp2, dst, R0, noreg);
514 }
515
516 /* ==== Apply load barrier, if required ==== */
517 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
518 assert(is_reference_type(type), "need_load_reference_barrier must check whether type is a reference type");
519
520 // If 'dst' clashes with either 'base' or 'ind_or_offs', use an intermediate result register
521 // to keep the values of those alive until the load reference barrier is applied.
522 Register intermediate_dst = (dst == base || (ind_or_offs.is_register() && dst == ind_or_offs.as_register()))
523 ? tmp2
524 : dst;
525
526 BarrierSetAssembler::load_at(masm, decorators, type,
527 base, ind_or_offs,
528 intermediate_dst,
529 tmp1, noreg,
530 preservation_level, L_handle_null);
531
532 load_reference_barrier(masm, decorators,
533 base, ind_or_offs,
534 intermediate_dst,
535 tmp1, R0,
536 preservation_level);
537
538 __ mr_if_needed(dst, intermediate_dst);
539 } else {
540 BarrierSetAssembler::load_at(masm, decorators, type,
541 base, ind_or_offs,
542 dst,
543 tmp1, tmp2,
544 preservation_level, L_handle_null);
545 }
546
547 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
548 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
549 if (ShenandoahSATBBarrier) {
550 __ block_comment("keep_alive_barrier (shenandoahgc) {");
551 satb_barrier_impl(masm, 0, noreg, noreg, dst, tmp1, tmp2, preservation_level);
552 __ block_comment("} keep_alive_barrier (shenandoahgc)");
553 }
554 }
555 }
556
557 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
558 assert(ShenandoahCardBarrier, "Should have been checked by caller");
559 assert_different_registers(base, tmp, R0);
560
561 if (ind_or_offs.is_constant()) {
562 __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
563 } else {
564 __ add(base, ind_or_offs.as_register(), base);
565 }
566
567 __ ld(tmp, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread); /* tmp = *[R16_thread + card_table_offset] */
568 __ srdi(base, base, CardTable::card_shift());
569 __ li(R0, CardTable::dirty_card_val());
570 __ stbx(R0, tmp, base);
571 }
572
573 // base: Base register of the reference's address.
574 // ind_or_offs: Index or offset of the reference's address.
575 // val: To-be-stored value/reference's new value.
576 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
577 Register base, RegisterOrConstant ind_or_offs, Register val,
578 Register tmp1, Register tmp2, Register tmp3,
579 MacroAssembler::PreservationLevel preservation_level) {
580 // 1: non-reference types require no barriers
581 if (!is_reference_type(type)) {
582 BarrierSetAssembler::store_at(masm, decorators, type,
583 base, ind_or_offs,
584 val,
585 tmp1, tmp2, tmp3,
586 preservation_level);
587 return;
588 }
589
590 bool storing_non_null = (val != noreg);
591
592 // 2: pre-barrier: SATB needs the previous value
593 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
594 satb_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
595 }
596
597 // Store!
598 BarrierSetAssembler::store_at(masm, decorators, type,
599 base, ind_or_offs,
600 val,
601 tmp1, tmp2, tmp3,
602 preservation_level);
603
604 // 3: post-barrier: card barrier needs store address
605 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
606 card_barrier(masm, base, ind_or_offs, tmp1);
607 }
608 }
609
610 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
611 Register dst, Register jni_env, Register obj,
612 Register tmp, Label &slowpath) {
613 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
614
615 assert_different_registers(jni_env, obj, tmp);
616
617 Label done;
618
619 // Fast path: Reference is null (JNI tags are zero for null pointers).
620 __ cmpdi(CR0, obj, 0);
621 __ beq(CR0, done);
622
623 // Resolve jobject using standard implementation.
624 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
625
626 // Check whether heap is stable.
627 __ lbz(tmp,
628 in_bytes(ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()),
629 jni_env);
630
631 __ andi_(tmp, tmp, ShenandoahHeap::EVACUATION | ShenandoahHeap::HAS_FORWARDED);
632 __ bne(CR0, slowpath);
633
634 __ bind(done);
635 __ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
636 }
637
638 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
639 Register obj, Register tmp, Label &slow_path) {
640 __ block_comment("try_peek_weak_handle_in_nmethod (shenandoahgc) {");
641
642 assert_different_registers(weak_handle, tmp, noreg);
643 assert_different_registers(obj, tmp, noreg);
644
645
646 Label done;
647
648 // Peek weak handle using the standard implementation.
649 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
650
651 // Check if the reference is null, and if it is, take the fast path.
652 __ cmpdi(CR0, obj, 0);
653 __ beq(CR0, done);
654
655 // Check if the heap is under weak-reference/roots processing, in
656 // which case we need to take the slow path.
657 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
658 __ andi_(tmp, tmp, ShenandoahHeap::WEAK_ROOTS);
659 __ bne(CR0, slow_path);
660 __ bind(done);
661
662 __ block_comment("} try_peek_weak_handle_in_nmethod (shenandoahgc)");
663 }
664
665 void ShenandoahBarrierSetAssembler::check_oop(MacroAssembler *masm, Register obj, const char* msg) {
666 if (!VerifyOops) {
667 return;
668 }
669
670 __ mr(R0, obj);
671
672 // This routine is sometimes called before applying GC barriers.
673 // With +COH, verification can touch the klass that may end up loading forwarding pointer instead.
674 Label L_skip;
675 if (UseCompactObjectHeaders) {
676 __ lbz(R0, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
677 __ andi_(R0, R0, ShenandoahHeap::HAS_FORWARDED);
678 __ bne(CR0, L_skip);
679 }
680
681 __ verify_oop(R0, msg);
682 __ bind(L_skip);
683 }
684
685 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
686 Register addr, Register count, Register preserve) {
687 assert(ShenandoahCardBarrier, "Should have been checked by caller");
688 assert_different_registers(addr, count, R0);
689
690 Label L_skip_loop, L_store_loop;
691
692 __ sldi_(count, count, LogBytesPerHeapOop);
693
694 // Zero length? Skip.
695 __ beq(CR0, L_skip_loop);
696
697 __ addi(count, count, -BytesPerHeapOop);
698 __ add(count, addr, count);
699 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
700 __ srdi(addr, addr, CardTable::card_shift());
701 __ srdi(count, count, CardTable::card_shift());
702 __ subf(count, addr, count);
703 __ ld(R0, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
704 __ add(addr, addr, R0);
705 __ addi(count, count, 1);
706 __ li(R0, 0);
707 __ mtctr(count);
708
709 // Byte store loop
710 __ bind(L_store_loop);
711 __ stb(R0, 0, addr);
712 __ addi(addr, addr, 1);
713 __ bdnz(L_store_loop);
714 __ bind(L_skip_loop);
715 }
716
717 #undef __
718
719 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
720 NativeInstruction* ni = nativeInstruction_at(pc);
721 assert(ni->is_jump(), "Must be");
722 NativeGeneralJump* jmp = nativeGeneralJump_at(pc);
723 return jmp->jump_destination();
724 }
725
726 static uint32_t encode_patchable_nop() {
727 return 0x60000000;
728 }
729
730 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
731 *((uint32_t*)pc) = encode_patchable_nop();
732 }
733
734 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
735 CodeBuffer cb(pc, BytesPerInstWord + 1);
736 MacroAssembler a(&cb);
737 a.b(target_pc);
738 }
739
740 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
741 return *((uint32_t*)pc) == encode_patchable_nop();
742 }
743
744 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
745 NativeInstruction* ni = nativeInstruction_at(pc);
746 return ni->is_jump() && nativeGeneralJump_at(pc)->jump_destination() == target_pc;
747 }
748
749 #ifdef COMPILER1
750
751 #define __ ce->masm()->
752
753 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
754 __ block_comment("keepalive_barrier_stub (shenandoahgc) {");
755 __ bind(*stub->entry());
756
757 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
758
759 Register obj = stub->obj()->as_register();
760
761 // If 'do_load()' returns false, the to-be-stored value is already available in 'obj'
762 if (stub->do_load()) {
763 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false);
764 }
765
766 // Fast path: reference is null.
767 __ cmpdi(CR0, obj, 0);
768 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
769
770 // Argument passing via the stack.
771 __ std(obj, -8, R1_SP);
772
773 address blob_addr = bs->keepalive_barrier_stub();
774 __ load_const_optimized(R0, blob_addr);
775 __ call_stub(R0);
776
777 __ b(*stub->continuation());
778 __ block_comment("} keepalive_barrier_stub (shenandoahgc)");
779 }
780
781 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
782 __ block_comment("load_reference_barrier_stub (shenandoahgc) {");
783
784 __ bind(*stub->entry());
785
786 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
787
788 Register obj = stub->obj()->as_register();
789 Register addr = stub->addr()->as_pointer_register();
790 Register slow_result = stub->slow_result()->as_register();
791 assert_different_registers(obj, addr, slow_result);
792 assert(slow_result == R3_RET, "C1 must know about our slow call result register");
793
794 // Argument passing via the stack.
795 __ std(obj, -8, R1_SP);
796 __ std(addr, -16, R1_SP);
797
798 address blob_addr = bs->load_reference_barrier_stub(stub->decorators());
799 __ load_const_optimized(R0, blob_addr);
800 __ call_stub(R0);
801 if (obj != slow_result) {
802 __ mr(obj, slow_result);
803 }
804
805 __ b(*stub->continuation());
806 __ block_comment("} load_reference_barrier_stub (shenandoahgc)");
807 }
808
809 #undef __
810
811 #define __ sasm->
812
813 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_runtime_stub(StubAssembler* sasm) {
814 __ block_comment("keepalive_barrier_runtime_stub (shenandoahgc) {");
815
816 Register obj = R3_ARG1;
817 Register tmp1 = R11_scratch1;
818 Register tmp2 = R12_scratch2;
819
820 // Save registers we are about to clobber
821 __ std(obj, -16, R1_SP);
822 __ std(tmp1, -24, R1_SP);
823 __ std(tmp2, -32, R1_SP);
824
825 // Pull the arguments from stack
826 __ ld(obj, -8, R1_SP);
827
828 satb_barrier(sasm, noreg, noreg, obj, tmp1, tmp2, MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS, 4 * BytesPerWord);
829
830 // Restore registers
831 __ ld(tmp2, -32, R1_SP);
832 __ ld(tmp1, -24, R1_SP);
833 __ ld(obj, -16, R1_SP);
834
835 __ blr();
836 __ block_comment("} keepalive_barrier_runtime_stub (shenandoahgc)");
837 }
838
839 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
840 __ block_comment("load_reference_barrier_runtime_stub (shenandoahgc) {");
841
842 Register obj = R3_ARG1;
843 Register addr = R4_ARG2;
844 Register tmp1 = R11_scratch1;
845 Register tmp2 = R12_scratch2;
846
847 // Save registers we are about to clobber
848 __ std(addr, -24, R1_SP);
849 __ std(tmp1, -32, R1_SP);
850 __ std(tmp2, -40, R1_SP);
851
852 // Pull the arguments from the stack
853 __ ld(obj, -8, R1_SP);
854 __ ld(addr, -16, R1_SP);
855
856 load_reference_barrier(sasm, decorators, addr, noreg, obj, tmp1, tmp2,
857 MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS, 5 * BytesPerWord);
858
859 // Restore registers
860 __ ld(tmp2, -40, R1_SP);
861 __ ld(tmp1, -32, R1_SP);
862 __ ld(addr, -24, R1_SP);
863
864 __ blr();
865 __ block_comment("} load_reference_barrier_runtime_stub (shenandoahgc)");
866 }
867
868 #undef __
869
870 #endif // COMPILER1
871
872 #ifdef COMPILER2
873
874 #undef __
875 #define __ masm->
876
877 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Register addr, int disp, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
878 if (is_narrow) {
879 __ lwz(dst, disp, addr);
880 } else {
881 __ ld(dst, disp, addr);
882 }
883 if (is_acquire) {
884 __ twi_0(dst);
885 __ isync();
886 }
887
888 ShenandoahBarrierStubC2::load_post(masm, node, dst, Address(addr, disp), tmp1, tmp2, is_narrow);
889 }
890
891 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
892 Register dst, int disp, bool dst_narrow, Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
893
894 ShenandoahBarrierStubC2::store_pre(masm, node, Address(dst, disp), tmp1, tmp2, tmp3, dst_narrow);
895
896 if (dst_narrow && !src_narrow) {
897 // Need to encode into tmp, because we cannot clobber src.
898 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
899 src = __ encode_heap_oop(tmp1, src);
900 } else {
901 src = __ encode_heap_oop_not_null(tmp1, src);
902 }
903 }
904 if (dst_narrow) {
905 __ stw(src, disp, dst);
906 } else {
907 __ std(src, disp, dst);
908 }
909
910 ShenandoahBarrierStubC2::store_post(masm, node, Address(dst, disp), tmp1, tmp2);
911 }
912
913 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
914 Register oldval, Register newval, Register tmp1, Register tmp2, bool exchange, bool narrow, bool weak, bool acquire) {
915
916 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, res, tmp1, tmp2, narrow);
917
918 Register dest_current = exchange ? res : R0;
919 Label no_update;
920 int semantics = MacroAssembler::MemBarNone;
921
922 if (acquire) {
923 semantics = support_IRIW_for_not_multiple_copy_atomic_cpu ?
924 MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter;
925 }
926
927 if (!exchange) { __ li(res, 0); }
928 if (narrow) {
929 __ cmpxchgw(CR0, dest_current, oldval, newval, addr,
930 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
931 noreg, &no_update, true, weak);
932 } else {
933 __ cmpxchgd(CR0, dest_current, oldval, newval, addr,
934 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
935 noreg, &no_update, true, weak);
936 }
937 if (!exchange) { __ li(res, 1); }
938
939 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
940
941 __ bind(no_update);
942 }
943
944 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval, Register newval, Register addr, Register tmp1, Register tmp2) {
945 bool is_narrow = node->bottom_type()->isa_narrowoop();
946
947 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, preval, tmp1, tmp2, is_narrow);
948
949 if (is_narrow) {
950 __ getandsetw(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
951 } else {
952 __ getandsetd(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
953 }
954
955 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
956 __ isync();
957 } else {
958 __ sync();
959 }
960
961 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
962 }
963
964 #undef __
965 #define __ masm.
966
967 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
968 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
969 assert_different_registers(tmp1, tmp2, address.index(), address.base());
970
971 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
972 if (address.index() == noreg) {
973 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
974 } else {
975 __ add(tmp2, address.index(), address.base());
976 if (address.disp() != 0) {
977 __ addi(tmp2, tmp2, address.disp());
978 }
979 }
980 __ srdi(tmp2, tmp2, CardTable::card_shift());
981 __ li(R0, CardTable::dirty_card_val());
982 __ stbx(R0, tmp2, tmp1);
983 }
984
985 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Label* L_target) {
986 PhaseOutput* const output = Compile::current()->output();
987 if (output->in_scratch_emit_size()) {
988 // Avoid binding L_target in scratch emits.
989 // We know the patched check is exactly one instruction long.
990 __ nop();
991 return;
992 }
993
994 // Emit the unconditional branch in the first version of the method.
995 // Let the rest of runtime figure out how to manage it.
996 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
997 __ b(*L_target);
998 }
999
1000 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
1001 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1002 patchable_jump_if_gc_state(masm, test_state, entry());
1003 __ bind(*continuation());
1004 }
1005
1006 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1007 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1008 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1009
1010 __ bind(*entry());
1011
1012 // If we need to load ourselves, do it here.
1013 if (_do_load) {
1014 if (_narrow) {
1015 __ lwz(_obj, _addr.disp(), _addr.base());
1016 } else {
1017 __ ld(_obj, _addr.disp(), _addr.base());
1018 }
1019 }
1020
1021 // If the object is null, there is no point in applying barriers.
1022 maybe_far_jump_if_zero(masm, _obj);
1023
1024 // We need to make sure that loads done by callers survive across slow-path calls.
1025 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1026 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
1027 if (!_do_load || needs_both_barriers) {
1028 preserve(_obj);
1029 }
1030
1031 // Go for barriers. Barriers can return straight to continuation, as long
1032 // as another barrier is not needed and we can reach the fastpath.
1033 if (needs_both_barriers) {
1034 keepalive(masm, nullptr);
1035 lrb(masm);
1036 } else if (_needs_keep_alive_barrier) {
1037 keepalive(masm, continuation());
1038 } else if (_needs_load_ref_barrier) {
1039 lrb(masm);
1040 } else {
1041 ShouldNotReachHere();
1042 }
1043 }
1044
1045 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1046 __ cmpdi(CR0, reg, 0);
1047 // Branch to continuation if equal
1048 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
1049 }
1050
1051 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1052 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1053 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1054 Label L_through, L_slowpath;
1055
1056 // If another barrier is enabled as well, do a check for a specific barrier.
1057 if (_needs_load_ref_barrier) {
1058 assert(L_done == nullptr, "Should be");
1059 char state_to_check = ShenandoahHeap::MARKING;
1060 patchable_jump_if_not_gc_state(masm, state_to_check, &L_through);
1061 }
1062
1063 // Fast-path: put object into buffer.
1064 // If buffer is already full, go slow.
1065 __ ld(_tmp1, index_offset, R16_thread);
1066 __ cmpdi(CR0, _tmp1, 0);
1067 __ beq(CR0, L_slowpath);
1068 __ addi(_tmp1, _tmp1, -wordSize);
1069 __ std(_tmp1, index_offset, R16_thread);
1070 __ ld(_tmp2, buffer_offset, R16_thread);
1071
1072 // Store the object in queue.
1073 // If object is narrow, we need to decode it before inserting.
1074 if (_narrow) {
1075 __ add(_tmp2, _tmp2, _tmp1);
1076 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1077 __ stdx(decoded, _tmp2);
1078 } else {
1079 __ stdx(_obj, _tmp2, _tmp1);
1080 }
1081
1082 // Fast-path exits here.
1083 if (L_done != nullptr) {
1084 __ b(*L_done);
1085 } else {
1086 __ b(L_through);
1087 }
1088
1089 // Slow-path: call runtime to handle.
1090 __ bind(L_slowpath);
1091
1092 {
1093 SaveLiveRegisters slr(&masm, this);
1094
1095 // Go to runtime and handle the rest there.
1096 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1097 }
1098
1099 if (L_done != nullptr) {
1100 __ b(*L_done);
1101 } else {
1102 __ bind(L_through);
1103 }
1104 }
1105
1106 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1107 Label L_slow;
1108
1109 // If another barrier is enabled as well, do a check for a specific barrier.
1110 if (_needs_keep_alive_barrier) {
1111 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1112 patchable_jump_if_not_gc_state(masm, state_to_check, continuation());
1113 }
1114
1115 // If weak references are being processed, weak/phantom loads need to go slow,
1116 // regardless of their cset status.
1117 if (_needs_load_ref_weak_barrier) {
1118 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
1119 patchable_jump_if_gc_state(masm, state_to_check, &L_slow);
1120 }
1121
1122 // Cset-check. Fall-through to slow if in collection set.
1123 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1124 if (_narrow) {
1125 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1126 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1127 } else {
1128 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1129 }
1130 __ lbzx(_tmp2, _tmp2, _tmp1);
1131 maybe_far_jump_if_zero(masm, _tmp2);
1132
1133 // Slow path
1134 __ bind(L_slow);
1135
1136 // Obj is the result, need to temporarily stop preserving it.
1137 bool is_obj_preserved = is_preserved(_obj);
1138 if (is_obj_preserved) {
1139 dont_preserve(_obj);
1140 }
1141 {
1142 SaveLiveRegisters slr(&masm, this);
1143
1144 // Shuffle in the arguments. The end result should be:
1145 // c_rarg0 <-- obj
1146 // c_rarg1 <-- lea(addr)
1147 Register c_rarg0 = R3_ARG1;
1148 Register c_rarg1 = R4_ARG2;
1149 if (c_rarg0 == _obj) {
1150 __ addi(c_rarg1, _addr.base(), _addr.disp());
1151 } else if (c_rarg1 == _obj) {
1152 __ mr(_tmp1, c_rarg1);
1153 __ addi(c_rarg1, _addr.base(), _addr.disp());
1154 __ mr(c_rarg0, _tmp1);
1155 } else {
1156 assert_different_registers(c_rarg1, _obj);
1157 __ addi(c_rarg1, _addr.base(), _addr.disp());
1158 __ mr(c_rarg0, _obj);
1159 }
1160
1161 // Go to runtime and handle the rest there.
1162 __ call_VM_leaf(lrb_runtime_entry_addr(), c_rarg0, c_rarg1);
1163
1164 // Save the result where needed.
1165 if (_obj != R3_RET) {
1166 __ mr(_obj, R3_RET);
1167 }
1168 }
1169 if (is_obj_preserved) {
1170 preserve(_obj);
1171 }
1172
1173 __ b(*continuation());
1174 }
1175
1176 int ShenandoahBarrierStubC2::available_gp_registers() {
1177 Unimplemented(); // Not used
1178 return 0;
1179 }
1180
1181 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1182 Unimplemented(); // Not used
1183 return true;
1184 }
1185
1186 void ShenandoahBarrierStubC2::post_init() {
1187 // Do nothing.
1188 }
1189
1190 #endif // COMPILER2