1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2025, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahRuntime.hpp"
38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
39 #include "interpreter/interpreter.hpp"
40 #include "macroAssembler_ppc.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "utilities/globalDefinitions.hpp"
44 #include "vm_version_ppc.hpp"
45 #ifdef COMPILER1
46 #include "c1/c1_LIRAssembler.hpp"
47 #include "c1/c1_MacroAssembler.hpp"
48 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
49 #endif
50 #ifdef COMPILER2
51 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
52 #endif
53
54 #define __ masm->
55
56 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
57 Register base, RegisterOrConstant ind_or_offs,
58 Register tmp1, Register tmp2, Register tmp3,
59 MacroAssembler::PreservationLevel preservation_level) {
60 if (ShenandoahSATBBarrier) {
61 __ block_comment("satb_barrier (shenandoahgc) {");
62 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
63 __ block_comment("} satb_barrier (shenandoahgc)");
64 }
65 }
66
67 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
68 Register base, RegisterOrConstant ind_or_offs,
69 Register dst,
70 Register tmp1, Register tmp2,
71 MacroAssembler::PreservationLevel preservation_level) {
72 if (ShenandoahLoadRefBarrier) {
73 __ block_comment("load_reference_barrier (shenandoahgc) {");
74 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
75 __ block_comment("} load_reference_barrier (shenandoahgc)");
76 }
77 }
78
79 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
80 Register src, Register dst, Register count,
81 Register preserve1, Register preserve2) {
82 Register R11_tmp = R11_scratch1;
83
84 assert_different_registers(src, dst, count, R11_tmp, noreg);
85 if (preserve1 != noreg) {
86 // Technically not required, but likely to indicate an error.
87 assert_different_registers(preserve1, preserve2);
88 }
89
90 /* ==== Check whether barrier is required (optimizations) ==== */
91 // Fast path: Component type of array is not a reference type.
92 if (!is_reference_type(type)) {
93 return;
94 }
95
96 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
97
98 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
99 // any useful information.
100 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahLoadRefBarrier) {
101 return;
102 }
103
104 __ block_comment("arraycopy_prologue (shenandoahgc) {");
105 Label skip_prologue;
106
107 // Fast path: Array is of length zero.
108 __ cmpdi(CR0, count, 0);
109 __ beq(CR0, skip_prologue);
110
111 /* ==== Check whether barrier is required (gc state) ==== */
112 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
113 R16_thread);
114
115 // The set of garbage collection states requiring barriers depends on the available barrier types and the
116 // type of the reference in question.
117 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
118 // for the garbage collector.
119 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
120 ? ShenandoahHeap::HAS_FORWARDED
121 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
122
123 __ andi_(R11_tmp, R11_tmp, required_states);
124 __ beq(CR0, skip_prologue);
125
126 /* ==== Invoke runtime ==== */
127 // Save to-be-preserved registers.
128 int highest_preserve_register_index = 0;
129 {
130 if (preserve1 != noreg && preserve1->is_volatile()) {
131 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
132 }
133 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
134 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
135 }
136
137 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
138 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
139 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
140
141 __ save_LR(R11_tmp);
142 __ push_frame_reg_args(-BytesPerWord * highest_preserve_register_index,
143 R11_tmp);
144 }
145
146 // Invoke runtime.
147 address jrt_address = nullptr;
148 if (UseCompressedOops) {
149 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
150 } else {
151 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
152 }
153 assert(jrt_address != nullptr, "jrt routine cannot be found");
154
155 __ call_VM_leaf(jrt_address, src, dst, count);
156
157 // Restore to-be-preserved registers.
158 {
159 __ pop_frame();
160 __ restore_LR(R11_tmp);
161
162 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
163 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
164 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
165
166 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
167 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
168 }
169 if (preserve1 != noreg && preserve1->is_volatile()) {
170 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
171 }
172 }
173
174 __ bind(skip_prologue);
175 __ block_comment("} arraycopy_prologue (shenandoahgc)");
176 }
177
178 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
179 Register dst, Register count,
180 Register preserve) {
181 if (ShenandoahCardBarrier && is_reference_type(type)) {
182 __ block_comment("arraycopy_epilogue (shenandoahgc) {");
183 gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
184 __ block_comment("} arraycopy_epilogue (shenandoahgc)");
185 }
186 }
187
188 // The to-be-enqueued value can either be determined
189 // - dynamically by passing the reference's address information (load mode) or
190 // - statically by passing a register the value is stored in (preloaded mode)
191 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
192 // - for incremental-update barriers.
193 //
194 // decorators: The previous value's decorator set.
195 // In "load mode", the value must equal '0'.
196 // base: Base register of the reference's address (load mode).
197 // In "preloaded mode", the register must equal 'noreg'.
198 // ind_or_offs: Index or offset of the reference's address (load mode).
199 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
200 // pre_val: Register holding the to-be-stored value (preloaded mode).
201 // In "load mode", this register acts as a temporary register and must
202 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
203 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
204 void ShenandoahBarrierSetAssembler::satb_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
205 Register base, RegisterOrConstant ind_or_offs,
206 Register pre_val,
207 Register tmp1, Register tmp2,
208 MacroAssembler::PreservationLevel preservation_level) {
209 assert(ShenandoahSATBBarrier, "Should be checked by caller");
210 assert_different_registers(tmp1, tmp2, pre_val, noreg);
211
212 Label skip_barrier;
213
214 /* ==== Determine necessary runtime invocation preservation measures ==== */
215 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
216 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
217 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
218
219 // Check whether marking is active.
220 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
221
222 __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
223 __ beq(CR0, skip_barrier);
224
225 /* ==== Determine the reference's previous value ==== */
226 bool preloaded_mode = base == noreg;
227 Register pre_val_save = noreg;
228
229 if (preloaded_mode) {
230 // Previous value has been passed to the method, so it must not be determined manually.
231 // In case 'pre_val' is a volatile register, it must be saved across the C-call
232 // as callers may depend on its value.
233 // Unless the general purposes registers are saved anyway, one of the temporary registers
234 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
235 if (!preserve_gp_registers && pre_val->is_volatile()) {
236 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
237 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
238 }
239
240 if ((decorators & IS_NOT_NULL) != 0) {
241 #ifdef ASSERT
242 __ cmpdi(CR0, pre_val, 0);
243 __ asm_assert_ne("null oop is not allowed");
244 #endif // ASSERT
245 } else {
246 __ cmpdi(CR0, pre_val, 0);
247 __ beq(CR0, skip_barrier);
248 }
249 } else {
250 // Load from the reference address to determine the reference's current value (before the store is being performed).
251 // Contrary to the given value in "preloaded mode", it is not necessary to preserve it.
252 assert(decorators == 0, "decorator set must be empty");
253 assert(base != noreg, "base must be a register");
254 assert(!ind_or_offs.is_register() || ind_or_offs.as_register() != noreg, "ind_or_offs must be a register");
255 if (UseCompressedOops) {
256 __ lwz(pre_val, ind_or_offs, base);
257 } else {
258 __ ld(pre_val, ind_or_offs, base);
259 }
260
261 __ cmpdi(CR0, pre_val, 0);
262 __ beq(CR0, skip_barrier);
263
264 if (UseCompressedOops) {
265 __ decode_heap_oop_not_null(pre_val);
266 }
267 }
268
269 /* ==== Try to enqueue the to-be-stored value directly into thread's local SATB mark queue ==== */
270 {
271 Label runtime;
272 Register Rbuffer = tmp1, Rindex = tmp2;
273
274 // Check whether the queue has enough capacity to store another oop.
275 // If not, jump to the runtime to commit the buffer and to allocate a new one.
276 // (The buffer's index corresponds to the amount of remaining free space.)
277 __ ld(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
278 __ cmpdi(CR0, Rindex, 0);
279 __ beq(CR0, runtime); // If index == 0 (buffer is full), goto runtime.
280
281 // Capacity suffices. Decrement the queue's size by the size of one oop.
282 // (The buffer is filled contrary to the heap's growing direction, i.e., it is filled downwards.)
283 __ addi(Rindex, Rindex, -wordSize);
284 __ std(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
285
286 // Enqueue the previous value and skip the invocation of the runtime.
287 __ ld(Rbuffer, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
288 __ stdx(pre_val, Rbuffer, Rindex);
289 __ b(skip_barrier);
290
291 __ bind(runtime);
292 }
293
294 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
295 // Save to-be-preserved registers.
296 int nbytes_save = 0;
297
298 if (needs_frame) {
299 if (preserve_gp_registers) {
300 nbytes_save = (preserve_fp_registers
301 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
302 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
303 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
304 }
305
306 __ save_LR(tmp1);
307 __ push_frame_reg_args(nbytes_save, tmp2);
308 }
309
310 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
311 assert(pre_val_save != noreg, "nv_save must not be noreg");
312
313 // 'pre_val' register must be saved manually unless general-purpose are preserved in general.
314 __ mr(pre_val_save, pre_val);
315 }
316
317 // Invoke runtime.
318 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
319
320 // Restore to-be-preserved registers.
321 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
322 __ mr(pre_val, pre_val_save);
323 }
324
325 if (needs_frame) {
326 __ pop_frame();
327 __ restore_LR(tmp1);
328
329 if (preserve_gp_registers) {
330 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
331 }
332 }
333
334 __ bind(skip_barrier);
335 }
336
337 // base: Base register of the reference's address.
338 // ind_or_offs: Index or offset of the reference's address (load mode).
339 // dst: Reference's address. In case the object has been evacuated, this is the to-space version
340 // of that object.
341 void ShenandoahBarrierSetAssembler::load_reference_barrier_impl(
342 MacroAssembler *masm, DecoratorSet decorators,
343 Register base, RegisterOrConstant ind_or_offs,
344 Register dst,
345 Register tmp1, Register tmp2,
346 MacroAssembler::PreservationLevel preservation_level) {
347 if (ind_or_offs.is_register()) {
348 assert_different_registers(tmp1, tmp2, base, ind_or_offs.as_register(), dst, noreg);
349 } else {
350 assert_different_registers(tmp1, tmp2, base, dst, noreg);
351 }
352
353 Label skip_barrier;
354
355 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
356 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
357 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
358 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
359 bool is_narrow = UseCompressedOops && !is_native;
360
361 /* ==== Check whether heap is stable ==== */
362 __ lbz(tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
363
364 if (is_strong) {
365 // For strong references, the heap is considered stable if "has forwarded" is not active.
366 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
367 __ beq(CR0, skip_barrier);
368 #ifdef ASSERT
369 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
370 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
371 __ asm_assert_ne("'has forwarded' is missing");
372 #endif // ASSERT
373 } else {
374 // For all non-strong references, the heap is considered stable if not any of "has forwarded",
375 // "root set processing", and "weak reference processing" is active.
376 // The additional phase conditions are in place to avoid the resurrection of weak references (see JDK-8266440).
377 Label skip_fastpath;
378 __ andi_(tmp1, tmp2, ShenandoahHeap::WEAK_ROOTS);
379 __ bne(CR0, skip_fastpath);
380
381 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
382 __ beq(CR0, skip_barrier);
383 #ifdef ASSERT
384 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
385 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
386 __ asm_assert_ne("'has forwarded' is missing");
387 #endif // ASSERT
388
389 __ bind(skip_fastpath);
390 }
391
392 /* ==== Check whether region is in collection set ==== */
393 if (is_strong) {
394 // Shenandoah stores metadata on regions in a continuous area of memory in which a single byte corresponds to
395 // an entire region of the shenandoah heap. At present, only the least significant bit is of significance
396 // and indicates whether the region is part of the collection set.
397 //
398 // All regions are of the same size and are always aligned by a power of two.
399 // Any address can thus be shifted by a fixed number of bits to retrieve the address prefix shared by
400 // all objects within that region (region identification bits).
401 //
402 // | unused bits | region identification bits | object identification bits |
403 // (Region size depends on a couple of criteria, such as page size, user-provided arguments and the max heap size.
404 // The number of object identification bits can thus not be determined at compile time.)
405 //
406 // ------------------------------------------------------- <--- cs (collection set) base address
407 // | lost space due to heap space base address -> 'ShenandoahHeap::in_cset_fast_test_addr()'
408 // | (region identification bits contain heap base offset)
409 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
410 // | collection set in the proper -> shift: 'region_size_bytes_shift_jint()'
411 // |
412 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
413 // + number of regions
414 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
415 __ srdi(tmp1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
416 __ lbzx(tmp2, tmp1, tmp2);
417 __ andi_(tmp2, tmp2, 1);
418 __ beq(CR0, skip_barrier);
419 }
420
421 /* ==== Invoke runtime ==== */
422 // Save to-be-preserved registers.
423 int nbytes_save = 0;
424
425 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
426 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
427 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
428
429 if (needs_frame) {
430 if (preserve_gp_registers) {
431 nbytes_save = (preserve_fp_registers
432 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
433 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
434 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
435 }
436
437 __ save_LR(tmp1);
438 __ push_frame_reg_args(nbytes_save, tmp1);
439 }
440
441 // Calculate the reference's absolute address.
442 __ add(R4_ARG2, ind_or_offs, base);
443
444 // Invoke runtime.
445 address jrt_address = nullptr;
446
447 if (is_strong) {
448 if (is_narrow) {
449 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
450 } else {
451 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
452 }
453 } else if (is_weak) {
454 if (is_narrow) {
455 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
456 } else {
457 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
458 }
459 } else {
460 assert(is_phantom, "only remaining strength");
461 assert(!is_narrow, "phantom access cannot be narrow");
462 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
463 }
464 assert(jrt_address != nullptr, "jrt routine cannot be found");
465
466 __ call_VM_leaf(jrt_address, dst /* reference */, R4_ARG2 /* reference address */);
467
468 // Restore to-be-preserved registers.
469 if (preserve_gp_registers) {
470 __ mr(R0, R3_RET);
471 } else {
472 __ mr_if_needed(dst, R3_RET);
473 }
474
475 if (needs_frame) {
476 __ pop_frame();
477 __ restore_LR(tmp1);
478
479 if (preserve_gp_registers) {
480 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
481 __ mr(dst, R0);
482 }
483 }
484
485 __ bind(skip_barrier);
486 }
487
488 // base: Base register of the reference's address.
489 // ind_or_offs: Index or offset of the reference's address.
490 // L_handle_null: An optional label that will be jumped to if the reference is null.
491 void ShenandoahBarrierSetAssembler::load_at(
492 MacroAssembler *masm, DecoratorSet decorators, BasicType type,
493 Register base, RegisterOrConstant ind_or_offs, Register dst,
494 Register tmp1, Register tmp2,
495 MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
496 // Register must not clash, except 'base' and 'dst'.
497 if (ind_or_offs.is_register()) {
498 if (base != noreg) {
499 assert_different_registers(tmp1, tmp2, base, ind_or_offs.register_or_noreg(), R0, noreg);
500 }
501 assert_different_registers(tmp1, tmp2, dst, ind_or_offs.register_or_noreg(), R0, noreg);
502 } else {
503 if (base == noreg) {
504 assert_different_registers(tmp1, tmp2, base, R0, noreg);
505 }
506 assert_different_registers(tmp1, tmp2, dst, R0, noreg);
507 }
508
509 /* ==== Apply load barrier, if required ==== */
510 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
511 assert(is_reference_type(type), "need_load_reference_barrier must check whether type is a reference type");
512
513 // If 'dst' clashes with either 'base' or 'ind_or_offs', use an intermediate result register
514 // to keep the values of those alive until the load reference barrier is applied.
515 Register intermediate_dst = (dst == base || (ind_or_offs.is_register() && dst == ind_or_offs.as_register()))
516 ? tmp2
517 : dst;
518
519 BarrierSetAssembler::load_at(masm, decorators, type,
520 base, ind_or_offs,
521 intermediate_dst,
522 tmp1, noreg,
523 preservation_level, L_handle_null);
524
525 load_reference_barrier(masm, decorators,
526 base, ind_or_offs,
527 intermediate_dst,
528 tmp1, R0,
529 preservation_level);
530
531 __ mr_if_needed(dst, intermediate_dst);
532 } else {
533 BarrierSetAssembler::load_at(masm, decorators, type,
534 base, ind_or_offs,
535 dst,
536 tmp1, tmp2,
537 preservation_level, L_handle_null);
538 }
539
540 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
541 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
542 if (ShenandoahSATBBarrier) {
543 __ block_comment("keep_alive_barrier (shenandoahgc) {");
544 satb_barrier_impl(masm, 0, noreg, noreg, dst, tmp1, tmp2, preservation_level);
545 __ block_comment("} keep_alive_barrier (shenandoahgc)");
546 }
547 }
548 }
549
550 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
551 assert(ShenandoahCardBarrier, "Should have been checked by caller");
552 assert_different_registers(base, tmp, R0);
553
554 if (ind_or_offs.is_constant()) {
555 __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
556 } else {
557 __ add(base, ind_or_offs.as_register(), base);
558 }
559
560 __ ld(tmp, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread); /* tmp = *[R16_thread + card_table_offset] */
561 __ srdi(base, base, CardTable::card_shift());
562 __ li(R0, CardTable::dirty_card_val());
563 __ stbx(R0, tmp, base);
564 }
565
566 // base: Base register of the reference's address.
567 // ind_or_offs: Index or offset of the reference's address.
568 // val: To-be-stored value/reference's new value.
569 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
570 Register base, RegisterOrConstant ind_or_offs, Register val,
571 Register tmp1, Register tmp2, Register tmp3,
572 MacroAssembler::PreservationLevel preservation_level) {
573 // 1: non-reference types require no barriers
574 if (!is_reference_type(type)) {
575 BarrierSetAssembler::store_at(masm, decorators, type,
576 base, ind_or_offs,
577 val,
578 tmp1, tmp2, tmp3,
579 preservation_level);
580 return;
581 }
582
583 bool storing_non_null = (val != noreg);
584
585 // 2: pre-barrier: SATB needs the previous value
586 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
587 satb_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
588 }
589
590 // Store!
591 BarrierSetAssembler::store_at(masm, decorators, type,
592 base, ind_or_offs,
593 val,
594 tmp1, tmp2, tmp3,
595 preservation_level);
596
597 // 3: post-barrier: card barrier needs store address
598 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
599 card_barrier(masm, base, ind_or_offs, tmp1);
600 }
601 }
602
603 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
604 Register dst, Register jni_env, Register obj,
605 Register tmp, Label &slowpath) {
606 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
607
608 assert_different_registers(jni_env, obj, tmp);
609
610 Label done;
611
612 // Fast path: Reference is null (JNI tags are zero for null pointers).
613 __ cmpdi(CR0, obj, 0);
614 __ beq(CR0, done);
615
616 // Resolve jobject using standard implementation.
617 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
618
619 // Check whether heap is stable.
620 __ lbz(tmp,
621 in_bytes(ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()),
622 jni_env);
623
624 __ andi_(tmp, tmp, ShenandoahHeap::EVACUATION | ShenandoahHeap::HAS_FORWARDED);
625 __ bne(CR0, slowpath);
626
627 __ bind(done);
628 __ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
629 }
630
631 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
632 Register obj, Register tmp, Label &slow_path) {
633 __ block_comment("try_peek_weak_handle_in_nmethod (shenandoahgc) {");
634
635 assert_different_registers(weak_handle, tmp, noreg);
636 assert_different_registers(obj, tmp, noreg);
637
638
639 Label done;
640
641 // Peek weak handle using the standard implementation.
642 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
643
644 // Check if the reference is null, and if it is, take the fast path.
645 __ cmpdi(CR0, obj, 0);
646 __ beq(CR0, done);
647
648 // Check if the heap is under weak-reference/roots processing, in
649 // which case we need to take the slow path.
650 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
651 __ andi_(tmp, tmp, ShenandoahHeap::WEAK_ROOTS);
652 __ bne(CR0, slow_path);
653 __ bind(done);
654
655 __ block_comment("} try_peek_weak_handle_in_nmethod (shenandoahgc)");
656 }
657
658 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
659 Register addr, Register count, Register preserve) {
660 assert(ShenandoahCardBarrier, "Should have been checked by caller");
661 assert_different_registers(addr, count, R0);
662
663 Label L_skip_loop, L_store_loop;
664
665 __ sldi_(count, count, LogBytesPerHeapOop);
666
667 // Zero length? Skip.
668 __ beq(CR0, L_skip_loop);
669
670 __ addi(count, count, -BytesPerHeapOop);
671 __ add(count, addr, count);
672 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
673 __ srdi(addr, addr, CardTable::card_shift());
674 __ srdi(count, count, CardTable::card_shift());
675 __ subf(count, addr, count);
676 __ ld(R0, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
677 __ add(addr, addr, R0);
678 __ addi(count, count, 1);
679 __ li(R0, 0);
680 __ mtctr(count);
681
682 // Byte store loop
683 __ bind(L_store_loop);
684 __ stb(R0, 0, addr);
685 __ addi(addr, addr, 1);
686 __ bdnz(L_store_loop);
687 __ bind(L_skip_loop);
688 }
689
690 #undef __
691
692 #ifdef COMPILER1
693
694 #define __ ce->masm()->
695
696 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
697 __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
698
699 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
700 __ bind(*stub->entry());
701
702 // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
703 // This stub is the slowpath of that function.
704
705 assert(stub->pre_val()->is_register(), "pre_val must be a register");
706 Register pre_val = stub->pre_val()->as_register();
707
708 // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
709 // ("preloaded mode" of the store barrier).
710 if (stub->do_load()) {
711 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false);
712 }
713
714 // Fast path: Reference is null.
715 __ cmpdi(CR0, pre_val, 0);
716 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
717
718 // Argument passing via the stack.
719 __ std(pre_val, -8, R1_SP);
720
721 __ load_const_optimized(R0, bs->pre_barrier_c1_runtime_code_blob()->code_begin());
722 __ call_stub(R0);
723
724 __ b(*stub->continuation());
725 __ block_comment("} gen_pre_barrier_stub (shenandoahgc)");
726 }
727
728 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler *ce,
729 ShenandoahLoadReferenceBarrierStub *stub) {
730 __ block_comment("gen_load_reference_barrier_stub (shenandoahgc) {");
731
732 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
733 __ bind(*stub->entry());
734
735 Register obj = stub->obj()->as_register();
736 Register res = stub->result()->as_register();
737 Register addr = stub->addr()->as_pointer_register();
738 Register tmp1 = stub->tmp1()->as_register();
739 Register tmp2 = stub->tmp2()->as_register();
740 assert_different_registers(addr, res, tmp1, tmp2);
741
742 assert(R3_RET == res, "res must be r3");
743
744 if (res != obj) {
745 __ mr(res, obj);
746 }
747
748 DecoratorSet decorators = stub->decorators();
749
750 /* ==== Check whether region is in collection set ==== */
751 // GC status (unstable) has already been verified by 'ShenandoahBarrierSetC1::load_reference_barrier_impl'.
752 // This stub is the slowpath of that function.
753
754 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
755 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
756 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
757 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
758
759 if (is_strong) {
760 // Check whether object is in collection set.
761 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
762 __ srdi(tmp1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
763 __ lbzx(tmp2, tmp1, tmp2);
764
765 __ andi_(tmp2, tmp2, 1);
766 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
767 }
768
769 address blob_addr = nullptr;
770
771 if (is_strong) {
772 if (is_native) {
773 blob_addr = bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin();
774 } else {
775 blob_addr = bs->load_reference_barrier_strong_rt_code_blob()->code_begin();
776 }
777 } else if (is_weak) {
778 blob_addr = bs->load_reference_barrier_weak_rt_code_blob()->code_begin();
779 } else {
780 assert(is_phantom, "only remaining strength");
781 blob_addr = bs->load_reference_barrier_phantom_rt_code_blob()->code_begin();
782 }
783
784 assert(blob_addr != nullptr, "code blob cannot be found");
785
786 // Argument passing via the stack. 'obj' is passed implicitly (as asserted above).
787 __ std(addr, -8, R1_SP);
788
789 __ load_const_optimized(tmp1, blob_addr, tmp2);
790 __ call_stub(tmp1);
791
792 // 'res' is 'R3_RET'. The result is thus already in the correct register.
793
794 __ b(*stub->continuation());
795 __ block_comment("} gen_load_reference_barrier_stub (shenandoahgc)");
796 }
797
798 #undef __
799
800 #define __ sasm->
801
802 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler *sasm) {
803 __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
804
805 Label runtime, skip_barrier;
806 BarrierSet *bs = BarrierSet::barrier_set();
807
808 // Argument passing via the stack.
809 const int caller_stack_slots = 3;
810
811 Register R0_pre_val = R0;
812 __ ld(R0, -8, R1_SP);
813 Register R11_tmp1 = R11_scratch1;
814 __ std(R11_tmp1, -16, R1_SP);
815 Register R12_tmp2 = R12_scratch2;
816 __ std(R12_tmp2, -24, R1_SP);
817
818 /* ==== Check whether marking is active ==== */
819 // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
820 // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
821 __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
822
823 __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::MARKING);
824 __ beq(CR0, skip_barrier);
825
826 /* ==== Add previous value directly to thread-local SATB mark queue ==== */
827 // Check queue's capacity. Jump to runtime if no free slot is available.
828 __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
829 __ cmpdi(CR0, R12_tmp2, 0);
830 __ beq(CR0, runtime);
831
832 // Capacity suffices. Decrement the queue's size by one slot (size of one oop).
833 __ addi(R12_tmp2, R12_tmp2, -wordSize);
834 __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
835
836 // Enqueue the previous value and skip the runtime invocation.
837 __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
838 __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
839 __ b(skip_barrier);
840
841 __ bind(runtime);
842
843 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
844 // Save to-be-preserved registers.
845 const int nbytes_save = (MacroAssembler::num_volatile_regs + caller_stack_slots) * BytesPerWord;
846 __ save_volatile_gprs(R1_SP, -nbytes_save);
847 __ save_LR(R11_tmp1);
848 __ push_frame_reg_args(nbytes_save, R11_tmp1);
849
850 // Invoke runtime.
851 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), R0_pre_val);
852
853 // Restore to-be-preserved registers.
854 __ pop_frame();
855 __ restore_LR(R11_tmp1);
856 __ restore_volatile_gprs(R1_SP, -nbytes_save);
857
858 __ bind(skip_barrier);
859
860 // Restore spilled registers.
861 __ ld(R11_tmp1, -16, R1_SP);
862 __ ld(R12_tmp2, -24, R1_SP);
863
864 __ blr();
865 __ block_comment("} generate_c1_pre_barrier_runtime_stub (shenandoahgc)");
866 }
867
868 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler *sasm,
869 DecoratorSet decorators) {
870 __ block_comment("generate_c1_load_reference_barrier_runtime_stub (shenandoahgc) {");
871
872 // Argument passing via the stack.
873 const int caller_stack_slots = 1;
874
875 // Save to-be-preserved registers.
876 const int nbytes_save = (MacroAssembler::num_volatile_regs - 1 // 'R3_ARG1' is skipped
877 + caller_stack_slots) * BytesPerWord;
878 __ save_volatile_gprs(R1_SP, -nbytes_save, true, false);
879
880 // Load arguments from stack.
881 // No load required, as caller has already loaded obj into R3.
882 Register R3_obj = R3_ARG1;
883 Register R4_load_addr = R4_ARG2;
884 __ ld(R4_load_addr, -8, R1_SP);
885
886 Register R11_tmp = R11_scratch1;
887
888 /* ==== Invoke runtime ==== */
889 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
890 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
891 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
892 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
893
894 address jrt_address = nullptr;
895
896 if (is_strong) {
897 if (is_native) {
898 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
899 } else {
900 if (UseCompressedOops) {
901 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
902 } else {
903 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
904 }
905 }
906 } else if (is_weak) {
907 assert(!is_native, "weak load reference barrier must not be called off-heap");
908 if (UseCompressedOops) {
909 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
910 } else {
911 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
912 }
913 } else {
914 assert(is_phantom, "reference type must be phantom");
915 assert(is_native, "phantom load reference barrier must be called off-heap");
916 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
917 }
918 assert(jrt_address != nullptr, "load reference barrier runtime routine cannot be found");
919
920 __ save_LR(R11_tmp);
921 __ push_frame_reg_args(nbytes_save, R11_tmp);
922
923 // Invoke runtime. Arguments are already stored in the corresponding registers.
924 __ call_VM_leaf(jrt_address, R3_obj, R4_load_addr);
925
926 // Restore to-be-preserved registers.
927 __ pop_frame();
928 __ restore_LR(R11_tmp);
929 __ restore_volatile_gprs(R1_SP, -nbytes_save, true, false); // Skip 'R3_RET' register.
930
931 __ blr();
932 __ block_comment("} generate_c1_load_reference_barrier_runtime_stub (shenandoahgc)");
933 }
934
935 #undef __
936
937 #endif // COMPILER1
938
939 #ifdef COMPILER2
940
941 #undef __
942 #define __ masm->
943
944 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Register addr, int disp, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
945 if (is_narrow) {
946 __ lwz(dst, disp, addr);
947 } else {
948 __ ld(dst, disp, addr);
949 }
950 if (is_acquire) {
951 __ twi_0(dst);
952 __ isync();
953 }
954
955 ShenandoahBarrierStubC2::load_post(masm, node, dst, Address(addr, disp), tmp1, tmp2, is_narrow);
956 }
957
958 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
959 Register dst, int disp, bool dst_narrow, Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
960
961 ShenandoahBarrierStubC2::store_pre(masm, node, Address(dst, disp), tmp1, tmp2, tmp3, dst_narrow);
962
963 if (dst_narrow && !src_narrow) {
964 // Need to encode into tmp, because we cannot clobber src.
965 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
966 src = __ encode_heap_oop(tmp1, src);
967 } else {
968 src = __ encode_heap_oop_not_null(tmp1, src);
969 }
970 }
971 if (dst_narrow) {
972 __ stw(src, disp, dst);
973 } else {
974 __ std(src, disp, dst);
975 }
976
977 ShenandoahBarrierStubC2::store_post(masm, node, Address(dst, disp), tmp1, tmp2);
978 }
979
980 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
981 Register oldval, Register newval, Register tmp1, Register tmp2, bool exchange, bool narrow, bool weak, bool acquire) {
982
983 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, res, tmp1, tmp2, narrow);
984
985 Register dest_current = exchange ? res : R0;
986 Label no_update;
987 int semantics = MacroAssembler::MemBarNone;
988
989 if (acquire) {
990 semantics = support_IRIW_for_not_multiple_copy_atomic_cpu ?
991 MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter;
992 }
993
994 if (!exchange) { __ li(res, 0); }
995 if (narrow) {
996 __ cmpxchgw(CR0, dest_current, oldval, newval, addr,
997 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
998 noreg, &no_update, true, weak);
999 } else {
1000 __ cmpxchgd(CR0, dest_current, oldval, newval, addr,
1001 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
1002 noreg, &no_update, true, weak);
1003 }
1004 if (!exchange) { __ li(res, 1); }
1005
1006 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
1007
1008 __ bind(no_update);
1009 }
1010
1011 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval, Register newval, Register addr, Register tmp1, Register tmp2) {
1012 bool is_narrow = node->bottom_type()->isa_narrowoop();
1013
1014 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, preval, tmp1, tmp2, is_narrow);
1015
1016 if (is_narrow) {
1017 __ getandsetw(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
1018 } else {
1019 __ getandsetd(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
1020 }
1021
1022 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
1023 __ isync();
1024 } else {
1025 __ sync();
1026 }
1027
1028 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
1029 }
1030
1031 #undef __
1032 #define __ masm.
1033
1034 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
1035 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1036 assert_different_registers(tmp1, tmp2, address.index(), address.base());
1037
1038 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
1039 if (address.index() == noreg) {
1040 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
1041 } else {
1042 __ add(tmp2, address.index(), address.base());
1043 if (address.disp() != 0) {
1044 __ addi(tmp2, tmp2, address.disp());
1045 }
1046 }
1047 __ srdi(tmp2, tmp2, CardTable::card_shift());
1048 __ li(R0, CardTable::dirty_card_val());
1049 __ stbx(R0, tmp2, tmp1);
1050 }
1051
1052 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
1053 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1054
1055 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)), R16_thread);
1056 __ cmpdi(CR0, tmp, 0);
1057 // Branch to entry if not equal
1058 __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *entry());
1059 // This is were the slowpath stub will return to
1060 __ bind(*continuation());
1061 }
1062
1063 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1064 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1065 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1066
1067 __ bind(*entry());
1068
1069 // If we need to load ourselves, do it here.
1070 if (_do_load) {
1071 if (_narrow) {
1072 __ lwz(_obj, _addr.disp(), _addr.base());
1073 } else {
1074 __ ld(_obj, _addr.disp(), _addr.base());
1075 }
1076 }
1077
1078 // If the object is null, there is no point in applying barriers.
1079 maybe_far_jump_if_zero(masm, _obj);
1080
1081 // We need to make sure that loads done by callers survive across slow-path calls.
1082 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1083 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
1084 if (!_do_load || needs_both_barriers) {
1085 preserve(_obj);
1086 }
1087
1088 // Go for barriers. Barriers can return straight to continuation, as long
1089 // as another barrier is not needed and we can reach the fastpath.
1090 if (needs_both_barriers) {
1091 keepalive(masm, nullptr);
1092 lrb(masm);
1093 } else if (_needs_keep_alive_barrier) {
1094 keepalive(masm, continuation());
1095 } else if (_needs_load_ref_barrier) {
1096 lrb(masm);
1097 } else {
1098 ShouldNotReachHere();
1099 }
1100 }
1101
1102 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1103 __ cmpdi(CR0, reg, 0);
1104 // Branch to continuation if equal
1105 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
1106 }
1107
1108 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1109 const int gcstate_offset = in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING));
1110 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1111 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1112 Label L_through, L_slowpath;
1113
1114 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1115 if (_needs_load_ref_barrier) {
1116 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
1117 __ lbz(_tmp1, gcstate_offset, R16_thread);
1118 __ cmpdi(CR0, _tmp1, 0);
1119 __ beq(CR0, L_through);
1120 }
1121
1122 // Fast-path: put object into buffer.
1123 // If buffer is already full, go slow.
1124 __ ld(_tmp1, index_offset, R16_thread);
1125 __ cmpdi(CR0, _tmp1, 0);
1126 __ beq(CR0, L_slowpath);
1127 __ addi(_tmp1, _tmp1, -wordSize);
1128 __ std(_tmp1, index_offset, R16_thread);
1129 __ ld(_tmp2, buffer_offset, R16_thread);
1130
1131 // Store the object in queue.
1132 // If object is narrow, we need to decode it before inserting.
1133 if (_narrow) {
1134 __ add(_tmp2, _tmp2, _tmp1);
1135 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1136 __ stdx(decoded, _tmp2);
1137 } else {
1138 __ stdx(_obj, _tmp2, _tmp1);
1139 }
1140
1141 // Fast-path exits here.
1142 if (L_done != nullptr) {
1143 __ b(*L_done);
1144 } else {
1145 __ b(L_through);
1146 }
1147
1148 // Slow-path: call runtime to handle.
1149 __ bind(L_slowpath);
1150
1151 {
1152 SaveLiveRegisters slr(&masm, this);
1153
1154 // Go to runtime and handle the rest there.
1155 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1156 }
1157
1158 if (L_done != nullptr) {
1159 __ b(*L_done);
1160 } else {
1161 __ bind(L_through);
1162 }
1163 }
1164
1165 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1166 Label L_slow;
1167
1168 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1169 if (_needs_keep_alive_barrier) {
1170 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1171 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)), R16_thread);
1172 maybe_far_jump_if_zero(masm, _tmp1);
1173 }
1174
1175 // If weak references are being processed, weak/phantom loads need to go slow,
1176 // regardless of their cset status.
1177 if (_needs_load_ref_weak_barrier) {
1178 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)), R16_thread);
1179 __ cmpdi(CR0, _tmp1, 0);
1180 __ bne(CR0, L_slow);
1181 }
1182
1183 // Cset-check. Fall-through to slow if in collection set.
1184 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1185 if (_narrow) {
1186 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1187 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1188 } else {
1189 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1190 }
1191 __ lbzx(_tmp2, _tmp2, _tmp1);
1192 maybe_far_jump_if_zero(masm, _tmp2);
1193
1194 // Slow path
1195 __ bind(L_slow);
1196
1197 // Obj is the result, need to temporarily stop preserving it.
1198 bool is_obj_preserved = is_preserved(_obj);
1199 if (is_obj_preserved) {
1200 dont_preserve(_obj);
1201 }
1202 {
1203 SaveLiveRegisters slr(&masm, this);
1204
1205 // Shuffle in the arguments. The end result should be:
1206 // c_rarg0 <-- obj
1207 // c_rarg1 <-- lea(addr)
1208 Register c_rarg0 = R3_ARG1;
1209 Register c_rarg1 = R4_ARG2;
1210 if (c_rarg0 == _obj) {
1211 __ addi(c_rarg1, _addr.base(), _addr.disp());
1212 } else if (c_rarg1 == _obj) {
1213 __ mr(_tmp1, c_rarg1);
1214 __ addi(c_rarg1, _addr.base(), _addr.disp());
1215 __ mr(c_rarg0, _tmp1);
1216 } else {
1217 assert_different_registers(c_rarg1, _obj);
1218 __ addi(c_rarg1, _addr.base(), _addr.disp());
1219 __ mr(c_rarg0, _obj);
1220 }
1221
1222 // Go to runtime and handle the rest there.
1223 __ call_VM_leaf(lrb_runtime_entry_addr(), c_rarg0, c_rarg1);
1224
1225 // Save the result where needed.
1226 if (_obj != R3_RET) {
1227 __ mr(_obj, R3_RET);
1228 }
1229 }
1230 if (is_obj_preserved) {
1231 preserve(_obj);
1232 }
1233
1234 __ b(*continuation());
1235 }
1236
1237 int ShenandoahBarrierStubC2::available_gp_registers() {
1238 Unimplemented(); // Not used
1239 return 0;
1240 }
1241
1242 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1243 Unimplemented(); // Not used
1244 return true;
1245 }
1246
1247 void ShenandoahBarrierStubC2::post_init() {
1248 // Do nothing.
1249 }
1250
1251 #endif // COMPILER2