1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2025, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "macroAssembler_ppc.hpp"
42 #include "nativeInst_ppc.hpp"
43 #include "runtime/icache.hpp"
44 #include "runtime/javaThread.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "utilities/globalDefinitions.hpp"
47 #include "vm_version_ppc.hpp"
48 #ifdef COMPILER1
49 #include "c1/c1_LIRAssembler.hpp"
50 #include "c1/c1_MacroAssembler.hpp"
51 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
52 #endif
53 #ifdef COMPILER2
54 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
55 #include "opto/output.hpp"
56 #endif
57
58 #define __ masm->
59
60 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
61 Register base, RegisterOrConstant ind_or_offs,
62 Register tmp1, Register tmp2, Register tmp3,
63 MacroAssembler::PreservationLevel preservation_level,
64 int extra_stack_space) {
65 if (ShenandoahSATBBarrier) {
66 __ block_comment("satb_barrier (shenandoahgc) {");
67 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level, extra_stack_space);
68 __ block_comment("} satb_barrier (shenandoahgc)");
69 }
70 }
71
72 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
73 Register base, RegisterOrConstant ind_or_offs,
74 Register dst,
75 Register tmp1, Register tmp2,
76 MacroAssembler::PreservationLevel preservation_level,
77 int extra_stack_space) {
78 if (ShenandoahLoadRefBarrier) {
79 __ block_comment("load_reference_barrier (shenandoahgc) {");
80 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level, extra_stack_space);
81 __ block_comment("} load_reference_barrier (shenandoahgc)");
82 }
83 }
84
85 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
86 Register src, Register dst, Register count,
87 Register preserve1, Register preserve2) {
88 Register R11_tmp = R11_scratch1;
89
90 assert_different_registers(src, dst, count, R11_tmp, noreg);
91 if (preserve1 != noreg) {
92 // Technically not required, but likely to indicate an error.
93 assert_different_registers(preserve1, preserve2);
94 }
95
96 /* ==== Check whether barrier is required (optimizations) ==== */
97 // Fast path: Component type of array is not a reference type.
98 if (!is_reference_type(type)) {
99 return;
100 }
101
102 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
103
104 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
105 // any useful information.
106 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahLoadRefBarrier) {
107 return;
108 }
109
110 __ block_comment("arraycopy_prologue (shenandoahgc) {");
111 Label skip_prologue;
112
113 // Fast path: Array is of length zero.
114 __ cmpdi(CR0, count, 0);
115 __ beq(CR0, skip_prologue);
116
117 /* ==== Check whether barrier is required (gc state) ==== */
118 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
119 R16_thread);
120
121 // The set of garbage collection states requiring barriers depends on the available barrier types and the
122 // type of the reference in question.
123 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
124 // for the garbage collector.
125 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
126 ? ShenandoahHeap::HAS_FORWARDED
127 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
128
129 __ andi_(R11_tmp, R11_tmp, required_states);
130 __ beq(CR0, skip_prologue);
131
132 /* ==== Invoke runtime ==== */
133 // Save to-be-preserved registers.
134 int highest_preserve_register_index = 0;
135 {
136 if (preserve1 != noreg && preserve1->is_volatile()) {
137 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
138 }
139 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
140 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
141 }
142
143 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
144 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
145 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
146
147 __ save_LR(R11_tmp);
148 __ push_frame_reg_args(-BytesPerWord * highest_preserve_register_index,
149 R11_tmp);
150 }
151
152 // Invoke runtime.
153 address jrt_address = nullptr;
154 if (UseCompressedOops) {
155 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
156 } else {
157 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
158 }
159 assert(jrt_address != nullptr, "jrt routine cannot be found");
160
161 __ call_VM_leaf(jrt_address, src, dst, count);
162
163 // Restore to-be-preserved registers.
164 {
165 __ pop_frame();
166 __ restore_LR(R11_tmp);
167
168 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
169 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
170 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
171
172 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
173 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
174 }
175 if (preserve1 != noreg && preserve1->is_volatile()) {
176 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
177 }
178 }
179
180 __ bind(skip_prologue);
181 __ block_comment("} arraycopy_prologue (shenandoahgc)");
182 }
183
184 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
185 Register dst, Register count,
186 Register preserve) {
187 if (ShenandoahCardBarrier && is_reference_type(type)) {
188 __ block_comment("arraycopy_epilogue (shenandoahgc) {");
189 gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
190 __ block_comment("} arraycopy_epilogue (shenandoahgc)");
191 }
192 }
193
194 // The to-be-enqueued value can either be determined
195 // - dynamically by passing the reference's address information (load mode) or
196 // - statically by passing a register the value is stored in (preloaded mode)
197 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
198 // - for incremental-update barriers.
199 //
200 // decorators: The previous value's decorator set.
201 // In "load mode", the value must equal '0'.
202 // base: Base register of the reference's address (load mode).
203 // In "preloaded mode", the register must equal 'noreg'.
204 // ind_or_offs: Index or offset of the reference's address (load mode).
205 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
206 // pre_val: Register holding the to-be-stored value (preloaded mode).
207 // In "load mode", this register acts as a temporary register and must
208 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
209 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
210 void ShenandoahBarrierSetAssembler::satb_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
211 Register base, RegisterOrConstant ind_or_offs,
212 Register pre_val,
213 Register tmp1, Register tmp2,
214 MacroAssembler::PreservationLevel preservation_level,
215 int extra_stack_space) {
216 assert(ShenandoahSATBBarrier, "Should be checked by caller");
217 assert_different_registers(tmp1, tmp2, pre_val, noreg);
218
219 Label skip_barrier;
220
221 /* ==== Determine necessary runtime invocation preservation measures ==== */
222 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
223 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
224 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
225
226 // Check whether marking is active.
227 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
228
229 __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
230 __ beq(CR0, skip_barrier);
231
232 /* ==== Determine the reference's previous value ==== */
233 bool preloaded_mode = base == noreg;
234 Register pre_val_save = noreg;
235
236 if (preloaded_mode) {
237 // Previous value has been passed to the method, so it must not be determined manually.
238 // In case 'pre_val' is a volatile register, it must be saved across the C-call
239 // as callers may depend on its value.
240 // Unless the general purposes registers are saved anyway, one of the temporary registers
241 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
242 if (!preserve_gp_registers && pre_val->is_volatile()) {
243 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
244 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
245 }
246
247 if ((decorators & IS_NOT_NULL) != 0) {
248 #ifdef ASSERT
249 __ cmpdi(CR0, pre_val, 0);
250 __ asm_assert_ne("null oop is not allowed");
251 #endif // ASSERT
252 } else {
253 __ cmpdi(CR0, pre_val, 0);
254 __ beq(CR0, skip_barrier);
255 }
256 } else {
257 // Load from the reference address to determine the reference's current value (before the store is being performed).
258 // Contrary to the given value in "preloaded mode", it is not necessary to preserve it.
259 assert(decorators == 0, "decorator set must be empty");
260 assert(base != noreg, "base must be a register");
261 assert(!ind_or_offs.is_register() || ind_or_offs.as_register() != noreg, "ind_or_offs must be a register");
262 if (UseCompressedOops) {
263 __ lwz(pre_val, ind_or_offs, base);
264 } else {
265 __ ld(pre_val, ind_or_offs, base);
266 }
267
268 __ cmpdi(CR0, pre_val, 0);
269 __ beq(CR0, skip_barrier);
270
271 if (UseCompressedOops) {
272 __ decode_heap_oop_not_null(pre_val);
273 }
274 }
275
276 /* ==== Try to enqueue the to-be-stored value directly into thread's local SATB mark queue ==== */
277 {
278 Label runtime;
279 Register Rbuffer = tmp1, Rindex = tmp2;
280
281 // Check whether the queue has enough capacity to store another oop.
282 // If not, jump to the runtime to commit the buffer and to allocate a new one.
283 // (The buffer's index corresponds to the amount of remaining free space.)
284 __ ld(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
285 __ cmpdi(CR0, Rindex, 0);
286 __ beq(CR0, runtime); // If index == 0 (buffer is full), goto runtime.
287
288 // Capacity suffices. Decrement the queue's size by the size of one oop.
289 // (The buffer is filled contrary to the heap's growing direction, i.e., it is filled downwards.)
290 __ addi(Rindex, Rindex, -wordSize);
291 __ std(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
292
293 // Enqueue the previous value and skip the invocation of the runtime.
294 __ ld(Rbuffer, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
295 __ stdx(pre_val, Rbuffer, Rindex);
296 __ b(skip_barrier);
297
298 __ bind(runtime);
299 }
300
301 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
302 // Save to-be-preserved registers.
303 int nbytes_save = 0;
304
305 if (needs_frame) {
306 if (preserve_gp_registers) {
307 nbytes_save = (preserve_fp_registers
308 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
309 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord + extra_stack_space;
310 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
311 }
312
313 __ save_LR(tmp1);
314 __ push_frame_reg_args(nbytes_save, tmp2);
315 }
316
317 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
318 assert(pre_val_save != noreg, "nv_save must not be noreg");
319
320 // 'pre_val' register must be saved manually unless general-purpose are preserved in general.
321 __ mr(pre_val_save, pre_val);
322 }
323
324 // Invoke runtime.
325 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
326
327 // Restore to-be-preserved registers.
328 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
329 __ mr(pre_val, pre_val_save);
330 }
331
332 if (needs_frame) {
333 __ pop_frame();
334 __ restore_LR(tmp1);
335
336 if (preserve_gp_registers) {
337 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
338 }
339 }
340
341 __ bind(skip_barrier);
342 }
343
344 // base: Base register of the reference's address.
345 // ind_or_offs: Index or offset of the reference's address (load mode).
346 // dst: Reference's address. In case the object has been evacuated, this is the to-space version
347 // of that object.
348 void ShenandoahBarrierSetAssembler::load_reference_barrier_impl(
349 MacroAssembler *masm, DecoratorSet decorators,
350 Register base, RegisterOrConstant ind_or_offs,
351 Register dst,
352 Register tmp1, Register tmp2,
353 MacroAssembler::PreservationLevel preservation_level,
354 int extra_stack_space) {
355 if (ind_or_offs.is_register()) {
356 assert_different_registers(tmp1, tmp2, base, ind_or_offs.as_register(), dst, noreg);
357 } else {
358 assert_different_registers(tmp1, tmp2, base, dst, noreg);
359 }
360
361 Label skip_barrier;
362
363 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
364 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
365 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
366 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
367 bool is_narrow = UseCompressedOops && !is_native;
368
369 /* ==== Check whether heap is stable ==== */
370 __ lbz(tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
371
372 if (is_strong) {
373 // For strong references, the heap is considered stable if "has forwarded" is not active.
374 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
375 __ beq(CR0, skip_barrier);
376 #ifdef ASSERT
377 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
378 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
379 __ asm_assert_ne("'has forwarded' is missing");
380 #endif // ASSERT
381 } else {
382 // For all non-strong references, the heap is considered stable if not any of "has forwarded",
383 // "root set processing", and "weak reference processing" is active.
384 // The additional phase conditions are in place to avoid the resurrection of weak references (see JDK-8266440).
385 Label skip_fastpath;
386 __ andi_(tmp1, tmp2, ShenandoahHeap::WEAK_ROOTS);
387 __ bne(CR0, skip_fastpath);
388
389 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
390 __ beq(CR0, skip_barrier);
391 #ifdef ASSERT
392 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
393 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
394 __ asm_assert_ne("'has forwarded' is missing");
395 #endif // ASSERT
396
397 __ bind(skip_fastpath);
398 }
399
400 /* ==== Check whether region is in collection set ==== */
401 if (is_strong) {
402 // Shenandoah stores metadata on regions in a continuous area of memory in which a single byte corresponds to
403 // an entire region of the shenandoah heap. At present, only the least significant bit is of significance
404 // and indicates whether the region is part of the collection set.
405 //
406 // All regions are of the same size and are always aligned by a power of two.
407 // Any address can thus be shifted by a fixed number of bits to retrieve the address prefix shared by
408 // all objects within that region (region identification bits).
409 //
410 // | unused bits | region identification bits | object identification bits |
411 // (Region size depends on a couple of criteria, such as page size, user-provided arguments and the max heap size.
412 // The number of object identification bits can thus not be determined at compile time.)
413 //
414 // ------------------------------------------------------- <--- cs (collection set) base address
415 // | lost space due to heap space base address -> 'ShenandoahHeap::in_cset_fast_test_addr()'
416 // | (region identification bits contain heap base offset)
417 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
418 // | collection set in the proper -> shift: 'region_size_bytes_shift_jint()'
419 // |
420 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
421 // + number of regions
422 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
423 __ srdi(tmp1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
424 __ lbzx(tmp2, tmp1, tmp2);
425 __ andi_(tmp2, tmp2, 1);
426 __ beq(CR0, skip_barrier);
427 }
428
429 /* ==== Invoke runtime ==== */
430 // Save to-be-preserved registers.
431 int nbytes_save = 0;
432
433 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
434 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
435 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
436
437 if (needs_frame) {
438 if (preserve_gp_registers) {
439 nbytes_save = (preserve_fp_registers
440 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
441 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord + extra_stack_space;
442 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
443 }
444
445 __ save_LR(tmp1);
446 __ push_frame_reg_args(nbytes_save, tmp1);
447 }
448
449 // Calculate the reference's absolute address.
450 __ add(R4_ARG2, ind_or_offs, base);
451
452 // Invoke runtime.
453 address jrt_address = nullptr;
454
455 if (is_strong) {
456 if (is_narrow) {
457 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
458 } else {
459 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
460 }
461 } else if (is_weak) {
462 if (is_narrow) {
463 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
464 } else {
465 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
466 }
467 } else {
468 assert(is_phantom, "only remaining strength");
469 assert(!is_narrow, "phantom access cannot be narrow");
470 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
471 }
472 assert(jrt_address != nullptr, "jrt routine cannot be found");
473
474 __ call_VM_leaf(jrt_address, dst /* reference */, R4_ARG2 /* reference address */);
475
476 // Restore to-be-preserved registers.
477 if (preserve_gp_registers) {
478 __ mr(R0, R3_RET);
479 } else {
480 __ mr_if_needed(dst, R3_RET);
481 }
482
483 if (needs_frame) {
484 __ pop_frame();
485 __ restore_LR(tmp1);
486
487 if (preserve_gp_registers) {
488 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
489 __ mr(dst, R0);
490 }
491 }
492
493 __ bind(skip_barrier);
494 }
495
496 // base: Base register of the reference's address.
497 // ind_or_offs: Index or offset of the reference's address.
498 // L_handle_null: An optional label that will be jumped to if the reference is null.
499 void ShenandoahBarrierSetAssembler::load_at(
500 MacroAssembler *masm, DecoratorSet decorators, BasicType type,
501 Register base, RegisterOrConstant ind_or_offs, Register dst,
502 Register tmp1, Register tmp2,
503 MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
504 // Register must not clash, except 'base' and 'dst'.
505 if (ind_or_offs.is_register()) {
506 if (base != noreg) {
507 assert_different_registers(tmp1, tmp2, base, ind_or_offs.register_or_noreg(), R0, noreg);
508 }
509 assert_different_registers(tmp1, tmp2, dst, ind_or_offs.register_or_noreg(), R0, noreg);
510 } else {
511 if (base == noreg) {
512 assert_different_registers(tmp1, tmp2, base, R0, noreg);
513 }
514 assert_different_registers(tmp1, tmp2, dst, R0, noreg);
515 }
516
517 /* ==== Apply load barrier, if required ==== */
518 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
519 assert(is_reference_type(type), "need_load_reference_barrier must check whether type is a reference type");
520
521 // If 'dst' clashes with either 'base' or 'ind_or_offs', use an intermediate result register
522 // to keep the values of those alive until the load reference barrier is applied.
523 Register intermediate_dst = (dst == base || (ind_or_offs.is_register() && dst == ind_or_offs.as_register()))
524 ? tmp2
525 : dst;
526
527 BarrierSetAssembler::load_at(masm, decorators, type,
528 base, ind_or_offs,
529 intermediate_dst,
530 tmp1, noreg,
531 preservation_level, L_handle_null);
532
533 load_reference_barrier(masm, decorators,
534 base, ind_or_offs,
535 intermediate_dst,
536 tmp1, R0,
537 preservation_level);
538
539 __ mr_if_needed(dst, intermediate_dst);
540 } else {
541 BarrierSetAssembler::load_at(masm, decorators, type,
542 base, ind_or_offs,
543 dst,
544 tmp1, tmp2,
545 preservation_level, L_handle_null);
546 }
547
548 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
549 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
550 if (ShenandoahSATBBarrier) {
551 __ block_comment("keep_alive_barrier (shenandoahgc) {");
552 satb_barrier_impl(masm, 0, noreg, noreg, dst, tmp1, tmp2, preservation_level);
553 __ block_comment("} keep_alive_barrier (shenandoahgc)");
554 }
555 }
556 }
557
558 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
559 assert(ShenandoahCardBarrier, "Should have been checked by caller");
560 assert_different_registers(base, tmp, R0);
561
562 if (ind_or_offs.is_constant()) {
563 __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
564 } else {
565 __ add(base, ind_or_offs.as_register(), base);
566 }
567
568 __ ld(tmp, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread); /* tmp = *[R16_thread + card_table_offset] */
569 __ srdi(base, base, CardTable::card_shift());
570 __ li(R0, CardTable::dirty_card_val());
571 __ stbx(R0, tmp, base);
572 }
573
574 // base: Base register of the reference's address.
575 // ind_or_offs: Index or offset of the reference's address.
576 // val: To-be-stored value/reference's new value.
577 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
578 Register base, RegisterOrConstant ind_or_offs, Register val,
579 Register tmp1, Register tmp2, Register tmp3,
580 MacroAssembler::PreservationLevel preservation_level) {
581 // 1: non-reference types require no barriers
582 if (!is_reference_type(type)) {
583 BarrierSetAssembler::store_at(masm, decorators, type,
584 base, ind_or_offs,
585 val,
586 tmp1, tmp2, tmp3,
587 preservation_level);
588 return;
589 }
590
591 bool storing_non_null = (val != noreg);
592
593 // 2: pre-barrier: SATB needs the previous value
594 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
595 satb_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
596 }
597
598 // Store!
599 BarrierSetAssembler::store_at(masm, decorators, type,
600 base, ind_or_offs,
601 val,
602 tmp1, tmp2, tmp3,
603 preservation_level);
604
605 // 3: post-barrier: card barrier needs store address
606 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
607 card_barrier(masm, base, ind_or_offs, tmp1);
608 }
609 }
610
611 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
612 Register dst, Register jni_env, Register obj,
613 Register tmp, Label &slowpath) {
614 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
615
616 assert_different_registers(jni_env, obj, tmp);
617
618 Label done;
619
620 // Fast path: Reference is null (JNI tags are zero for null pointers).
621 __ cmpdi(CR0, obj, 0);
622 __ beq(CR0, done);
623
624 // Resolve jobject using standard implementation.
625 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
626
627 // Check whether heap is stable.
628 __ lbz(tmp,
629 in_bytes(ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()),
630 jni_env);
631
632 __ andi_(tmp, tmp, ShenandoahHeap::EVACUATION | ShenandoahHeap::HAS_FORWARDED);
633 __ bne(CR0, slowpath);
634
635 __ bind(done);
636 __ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
637 }
638
639 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
640 Register obj, Register tmp, Label &slow_path) {
641 __ block_comment("try_peek_weak_handle_in_nmethod (shenandoahgc) {");
642
643 assert_different_registers(weak_handle, tmp, noreg);
644 assert_different_registers(obj, tmp, noreg);
645
646
647 Label done;
648
649 // Peek weak handle using the standard implementation.
650 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
651
652 // Check if the reference is null, and if it is, take the fast path.
653 __ cmpdi(CR0, obj, 0);
654 __ beq(CR0, done);
655
656 // Check if the heap is under weak-reference/roots processing, in
657 // which case we need to take the slow path.
658 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
659 __ andi_(tmp, tmp, ShenandoahHeap::WEAK_ROOTS);
660 __ bne(CR0, slow_path);
661 __ bind(done);
662
663 __ block_comment("} try_peek_weak_handle_in_nmethod (shenandoahgc)");
664 }
665
666 void ShenandoahBarrierSetAssembler::check_oop(MacroAssembler *masm, Register obj, const char* msg) {
667 if (!VerifyOops) {
668 return;
669 }
670
671 __ mr(R0, obj);
672
673 // This routine is sometimes called before applying GC barriers.
674 // With +COH, verification can touch the klass that may end up loading forwarding pointer instead.
675 Label L_skip;
676 if (UseCompactObjectHeaders) {
677 __ lbz(R0, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
678 __ andi_(R0, R0, ShenandoahHeap::HAS_FORWARDED);
679 __ bne(CR0, L_skip);
680 }
681
682 __ verify_oop(R0, msg);
683 __ bind(L_skip);
684 }
685
686 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
687 Register addr, Register count, Register preserve) {
688 assert(ShenandoahCardBarrier, "Should have been checked by caller");
689 assert_different_registers(addr, count, R0);
690
691 Label L_skip_loop, L_store_loop;
692
693 __ sldi_(count, count, LogBytesPerHeapOop);
694
695 // Zero length? Skip.
696 __ beq(CR0, L_skip_loop);
697
698 __ addi(count, count, -BytesPerHeapOop);
699 __ add(count, addr, count);
700 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
701 __ srdi(addr, addr, CardTable::card_shift());
702 __ srdi(count, count, CardTable::card_shift());
703 __ subf(count, addr, count);
704 __ ld(R0, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
705 __ add(addr, addr, R0);
706 __ addi(count, count, 1);
707 __ li(R0, 0);
708 __ mtctr(count);
709
710 // Byte store loop
711 __ bind(L_store_loop);
712 __ stb(R0, 0, addr);
713 __ addi(addr, addr, 1);
714 __ bdnz(L_store_loop);
715 __ bind(L_skip_loop);
716 }
717
718 #undef __
719
720 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
721 NativeInstruction* ni = nativeInstruction_at(pc);
722 assert(ni->is_jump(), "Must be");
723 NativeGeneralJump* jmp = nativeGeneralJump_at(pc);
724 return jmp->jump_destination();
725 }
726
727 static uint32_t encode_patchable_nop() {
728 return 0x60000000;
729 }
730
731 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
732 *((uint32_t*)pc) = encode_patchable_nop();
733 ICache::invalidate_word(pc);
734 }
735
736 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
737 CodeBuffer cb(pc, BytesPerInstWord + 1);
738 MacroAssembler a(&cb);
739 a.b(target_pc);
740 ICache::invalidate_word(pc);
741 }
742
743 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
744 return *((uint32_t*)pc) == encode_patchable_nop();
745 }
746
747 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
748 NativeInstruction* ni = nativeInstruction_at(pc);
749 return ni->is_jump() && nativeGeneralJump_at(pc)->jump_destination() == target_pc;
750 }
751
752 #ifdef COMPILER1
753
754 #define __ ce->masm()->
755
756 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
757 __ block_comment("keepalive_barrier_stub (shenandoahgc) {");
758 __ bind(*stub->entry());
759
760 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
761
762 Register obj = stub->obj()->as_register();
763
764 // If 'do_load()' returns false, the to-be-stored value is already available in 'obj'
765 if (stub->do_load()) {
766 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false);
767 }
768
769 // Fast path: reference is null.
770 __ cmpdi(CR0, obj, 0);
771 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
772
773 // Argument passing via the stack.
774 __ std(obj, -8, R1_SP);
775
776 address blob_addr = bs->keepalive_barrier_stub();
777 __ load_const_optimized(R0, blob_addr);
778 __ call_stub(R0);
779
780 __ b(*stub->continuation());
781 __ block_comment("} keepalive_barrier_stub (shenandoahgc)");
782 }
783
784 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
785 __ block_comment("load_reference_barrier_stub (shenandoahgc) {");
786
787 __ bind(*stub->entry());
788
789 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
790
791 Register obj = stub->obj()->as_register();
792 Register addr = stub->addr()->as_pointer_register();
793 Register slow_result = stub->slow_result()->as_register();
794 assert_different_registers(obj, addr, slow_result);
795 assert(slow_result == R3_RET, "C1 must know about our slow call result register");
796
797 // Argument passing via the stack.
798 __ std(obj, -8, R1_SP);
799 __ std(addr, -16, R1_SP);
800
801 address blob_addr = bs->load_reference_barrier_stub(stub->decorators());
802 __ load_const_optimized(R0, blob_addr);
803 __ call_stub(R0);
804 if (obj != slow_result) {
805 __ mr(obj, slow_result);
806 }
807
808 __ b(*stub->continuation());
809 __ block_comment("} load_reference_barrier_stub (shenandoahgc)");
810 }
811
812 #undef __
813
814 #define __ sasm->
815
816 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_runtime_stub(StubAssembler* sasm) {
817 __ block_comment("keepalive_barrier_runtime_stub (shenandoahgc) {");
818
819 Register obj = R3_ARG1;
820 Register tmp1 = R11_scratch1;
821 Register tmp2 = R12_scratch2;
822
823 // Save registers we are about to clobber
824 __ std(obj, -16, R1_SP);
825 __ std(tmp1, -24, R1_SP);
826 __ std(tmp2, -32, R1_SP);
827
828 // Pull the arguments from stack
829 __ ld(obj, -8, R1_SP);
830
831 satb_barrier(sasm, noreg, noreg, obj, tmp1, tmp2, MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS, 4 * BytesPerWord);
832
833 // Restore registers
834 __ ld(tmp2, -32, R1_SP);
835 __ ld(tmp1, -24, R1_SP);
836 __ ld(obj, -16, R1_SP);
837
838 __ blr();
839 __ block_comment("} keepalive_barrier_runtime_stub (shenandoahgc)");
840 }
841
842 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
843 __ block_comment("load_reference_barrier_runtime_stub (shenandoahgc) {");
844
845 Register obj = R3_ARG1;
846 Register addr = R4_ARG2;
847 Register tmp1 = R11_scratch1;
848 Register tmp2 = R12_scratch2;
849
850 // Save registers we are about to clobber
851 __ std(addr, -24, R1_SP);
852 __ std(tmp1, -32, R1_SP);
853 __ std(tmp2, -40, R1_SP);
854
855 // Pull the arguments from the stack
856 __ ld(obj, -8, R1_SP);
857 __ ld(addr, -16, R1_SP);
858
859 load_reference_barrier(sasm, decorators, addr, noreg, obj, tmp1, tmp2,
860 MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS, 5 * BytesPerWord);
861
862 // Restore registers
863 __ ld(tmp2, -40, R1_SP);
864 __ ld(tmp1, -32, R1_SP);
865 __ ld(addr, -24, R1_SP);
866
867 __ blr();
868 __ block_comment("} load_reference_barrier_runtime_stub (shenandoahgc)");
869 }
870
871 #undef __
872
873 #endif // COMPILER1
874
875 #ifdef COMPILER2
876
877 #undef __
878 #define __ masm->
879
880 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Register addr, int disp, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
881 if (is_narrow) {
882 __ lwz(dst, disp, addr);
883 } else {
884 __ ld(dst, disp, addr);
885 }
886 if (is_acquire) {
887 __ twi_0(dst);
888 __ isync();
889 }
890
891 ShenandoahBarrierStubC2::load_post(masm, node, dst, Address(addr, disp), tmp1, tmp2, is_narrow);
892 }
893
894 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
895 Register dst, int disp, bool dst_narrow, Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
896
897 ShenandoahBarrierStubC2::store_pre(masm, node, Address(dst, disp), tmp1, tmp2, tmp3, dst_narrow);
898
899 if (dst_narrow && !src_narrow) {
900 // Need to encode into tmp, because we cannot clobber src.
901 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
902 src = __ encode_heap_oop(tmp1, src);
903 } else {
904 src = __ encode_heap_oop_not_null(tmp1, src);
905 }
906 }
907 if (dst_narrow) {
908 __ stw(src, disp, dst);
909 } else {
910 __ std(src, disp, dst);
911 }
912
913 ShenandoahBarrierStubC2::store_post(masm, node, Address(dst, disp), tmp1, tmp2);
914 }
915
916 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
917 Register oldval, Register newval, Register tmp1, Register tmp2, bool exchange, bool narrow, bool weak, bool acquire) {
918
919 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, res, tmp1, tmp2, narrow);
920
921 Register dest_current = exchange ? res : R0;
922 Label no_update;
923 int semantics = MacroAssembler::MemBarNone;
924
925 if (acquire) {
926 semantics = support_IRIW_for_not_multiple_copy_atomic_cpu ?
927 MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter;
928 }
929
930 if (!exchange) { __ li(res, 0); }
931 if (narrow) {
932 __ cmpxchgw(CR0, dest_current, oldval, newval, addr,
933 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
934 noreg, &no_update, true, weak);
935 } else {
936 __ cmpxchgd(CR0, dest_current, oldval, newval, addr,
937 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
938 noreg, &no_update, true, weak);
939 }
940 if (!exchange) { __ li(res, 1); }
941
942 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
943
944 __ bind(no_update);
945 }
946
947 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval, Register newval, Register addr, Register tmp1, Register tmp2) {
948 bool is_narrow = node->bottom_type()->isa_narrowoop();
949
950 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, preval, tmp1, tmp2, is_narrow);
951
952 if (is_narrow) {
953 __ getandsetw(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
954 } else {
955 __ getandsetd(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
956 }
957
958 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
959 __ isync();
960 } else {
961 __ sync();
962 }
963
964 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
965 }
966
967 #undef __
968 #define __ masm.
969
970 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
971 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
972 assert_different_registers(tmp1, tmp2, address.index(), address.base());
973
974 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
975 if (address.index() == noreg) {
976 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
977 } else {
978 __ add(tmp2, address.index(), address.base());
979 if (address.disp() != 0) {
980 __ addi(tmp2, tmp2, address.disp());
981 }
982 }
983 __ srdi(tmp2, tmp2, CardTable::card_shift());
984 __ li(R0, CardTable::dirty_card_val());
985 __ stbx(R0, tmp2, tmp1);
986 }
987
988 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Register tmp1, Register tmp2, Label* L_target) {
989 PhaseOutput* const output = Compile::current()->output();
990 if (output->in_scratch_emit_size()) {
991 // Avoid binding L_target in scratch emits.
992 // We know the patched check is exactly one instruction long.
993 __ nop();
994 return;
995 }
996
997 // Emit the unconditional branch in the first version of the method.
998 // Let the rest of runtime figure out how to manage it.
999 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
1000 __ b(*L_target);
1001 }
1002
1003 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp1, Register tmp2) {
1004 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1005 patchable_jump_if_gc_state(masm, test_state, tmp1, tmp2, entry());
1006 __ bind(*continuation());
1007 }
1008
1009 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1010 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1011 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1012
1013 __ bind(*entry());
1014
1015 // If we need to load ourselves, do it here.
1016 if (_do_load) {
1017 if (_narrow) {
1018 __ lwz(_obj, _addr.disp(), _addr.base());
1019 } else {
1020 __ ld(_obj, _addr.disp(), _addr.base());
1021 }
1022 }
1023
1024 // If the object is null, there is no point in applying barriers.
1025 maybe_far_jump_if_zero(masm, _obj);
1026
1027 // We need to make sure that loads done by callers survive across slow-path calls.
1028 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1029 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
1030 if (!_do_load || needs_both_barriers) {
1031 preserve(_obj);
1032 }
1033
1034 // Go for barriers. Barriers can return straight to continuation, as long
1035 // as another barrier is not needed and we can reach the fastpath.
1036 if (needs_both_barriers) {
1037 keepalive(masm, nullptr);
1038 lrb(masm);
1039 } else if (_needs_keep_alive_barrier) {
1040 keepalive(masm, continuation());
1041 } else if (_needs_load_ref_barrier) {
1042 lrb(masm);
1043 } else {
1044 ShouldNotReachHere();
1045 }
1046 }
1047
1048 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1049 __ cmpdi(CR0, reg, 0);
1050 // Branch to continuation if equal
1051 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
1052 }
1053
1054 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1055 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1056 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1057 Label L_through, L_slowpath;
1058
1059 // If another barrier is enabled as well, do a check for a specific barrier.
1060 if (_needs_load_ref_barrier) {
1061 assert(L_done == nullptr, "Should be");
1062 char state_to_check = ShenandoahHeap::MARKING;
1063 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_through);
1064 }
1065
1066 // Fast-path: put object into buffer.
1067 // If buffer is already full, go slow.
1068 __ ld(_tmp1, index_offset, R16_thread);
1069 __ cmpdi(CR0, _tmp1, 0);
1070 __ beq(CR0, L_slowpath);
1071 __ addi(_tmp1, _tmp1, -wordSize);
1072 __ std(_tmp1, index_offset, R16_thread);
1073 __ ld(_tmp2, buffer_offset, R16_thread);
1074
1075 // Store the object in queue.
1076 // If object is narrow, we need to decode it before inserting.
1077 if (_narrow) {
1078 __ add(_tmp2, _tmp2, _tmp1);
1079 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1080 __ stdx(decoded, _tmp2);
1081 } else {
1082 __ stdx(_obj, _tmp2, _tmp1);
1083 }
1084
1085 // Fast-path exits here.
1086 if (L_done != nullptr) {
1087 __ b(*L_done);
1088 } else {
1089 __ b(L_through);
1090 }
1091
1092 // Slow-path: call runtime to handle.
1093 __ bind(L_slowpath);
1094
1095 {
1096 SaveLiveRegisters slr(&masm, this);
1097
1098 // Go to runtime and handle the rest there.
1099 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1100 }
1101
1102 if (L_done != nullptr) {
1103 __ b(*L_done);
1104 } else {
1105 __ bind(L_through);
1106 }
1107 }
1108
1109 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1110 Label L_slow;
1111
1112 // If another barrier is enabled as well, do a check for a specific barrier.
1113 if (_needs_keep_alive_barrier) {
1114 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1115 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, continuation());
1116 }
1117
1118 // If weak references are being processed, weak/phantom loads need to go slow,
1119 // regardless of their cset status.
1120 if (_needs_load_ref_weak_barrier) {
1121 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
1122 patchable_jump_if_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_slow);
1123 }
1124
1125 // Cset-check. Fall-through to slow if in collection set.
1126 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1127 if (_narrow) {
1128 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1129 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1130 } else {
1131 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1132 }
1133 __ lbzx(_tmp2, _tmp2, _tmp1);
1134 maybe_far_jump_if_zero(masm, _tmp2);
1135
1136 // Slow path
1137 __ bind(L_slow);
1138
1139 // Obj is the result, need to temporarily stop preserving it.
1140 bool is_obj_preserved = is_preserved(_obj);
1141 if (is_obj_preserved) {
1142 dont_preserve(_obj);
1143 }
1144 {
1145 SaveLiveRegisters slr(&masm, this);
1146
1147 // Shuffle in the arguments. The end result should be:
1148 // c_rarg0 <-- obj
1149 // c_rarg1 <-- lea(addr)
1150 Register c_rarg0 = R3_ARG1;
1151 Register c_rarg1 = R4_ARG2;
1152 if (c_rarg0 == _obj) {
1153 __ addi(c_rarg1, _addr.base(), _addr.disp());
1154 } else if (c_rarg1 == _obj) {
1155 __ mr(_tmp1, c_rarg1);
1156 __ addi(c_rarg1, _addr.base(), _addr.disp());
1157 __ mr(c_rarg0, _tmp1);
1158 } else {
1159 assert_different_registers(c_rarg1, _obj);
1160 __ addi(c_rarg1, _addr.base(), _addr.disp());
1161 __ mr(c_rarg0, _obj);
1162 }
1163
1164 // Go to runtime and handle the rest there.
1165 __ call_VM_leaf(lrb_runtime_entry_addr(), c_rarg0, c_rarg1);
1166
1167 // Save the result where needed.
1168 if (_obj != R3_RET) {
1169 __ mr(_obj, R3_RET);
1170 }
1171 }
1172 if (is_obj_preserved) {
1173 preserve(_obj);
1174 }
1175
1176 __ b(*continuation());
1177 }
1178
1179 int ShenandoahBarrierStubC2::available_gp_registers() {
1180 Unimplemented(); // Not used
1181 return 0;
1182 }
1183
1184 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1185 Unimplemented(); // Not used
1186 return true;
1187 }
1188
1189 void ShenandoahBarrierStubC2::post_init() {
1190 // Do nothing.
1191 }
1192
1193 #endif // COMPILER2