1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2025, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahForwarding.hpp"
35 #include "gc/shenandoah/shenandoahHeap.hpp"
36 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
37 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "macroAssembler_ppc.hpp"
42 #include "runtime/javaThread.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/globalDefinitions.hpp"
45 #include "vm_version_ppc.hpp"
46 #ifdef COMPILER1
47 #include "c1/c1_LIRAssembler.hpp"
48 #include "c1/c1_MacroAssembler.hpp"
49 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
50 #endif
51 #ifdef COMPILER2
52 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
53 #endif
54
55 #define __ masm->
56
57 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
58 Register base, RegisterOrConstant ind_or_offs,
59 Register tmp1, Register tmp2, Register tmp3,
60 MacroAssembler::PreservationLevel preservation_level) {
61 if (ShenandoahSATBBarrier) {
62 __ block_comment("satb_barrier (shenandoahgc) {");
63 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
64 __ block_comment("} satb_barrier (shenandoahgc)");
65 }
66 }
67
68 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
69 Register base, RegisterOrConstant ind_or_offs,
70 Register dst,
71 Register tmp1, Register tmp2,
72 MacroAssembler::PreservationLevel preservation_level) {
73 if (ShenandoahLoadRefBarrier) {
74 __ block_comment("load_reference_barrier (shenandoahgc) {");
75 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
76 __ block_comment("} load_reference_barrier (shenandoahgc)");
77 }
78 }
79
80 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
81 Register src, Register dst, Register count,
82 Register preserve1, Register preserve2) {
83 Register R11_tmp = R11_scratch1;
84
85 assert_different_registers(src, dst, count, R11_tmp, noreg);
86 if (preserve1 != noreg) {
87 // Technically not required, but likely to indicate an error.
88 assert_different_registers(preserve1, preserve2);
89 }
90
91 /* ==== Check whether barrier is required (optimizations) ==== */
92 // Fast path: Component type of array is not a reference type.
93 if (!is_reference_type(type)) {
94 return;
95 }
96
97 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
98
99 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
100 // any useful information.
101 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahLoadRefBarrier) {
102 return;
103 }
104
105 __ block_comment("arraycopy_prologue (shenandoahgc) {");
106 Label skip_prologue;
107
108 // Fast path: Array is of length zero.
109 __ cmpdi(CR0, count, 0);
110 __ beq(CR0, skip_prologue);
111
112 /* ==== Check whether barrier is required (gc state) ==== */
113 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
114 R16_thread);
115
116 // The set of garbage collection states requiring barriers depends on the available barrier types and the
117 // type of the reference in question.
118 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
119 // for the garbage collector.
120 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
121 ? ShenandoahHeap::HAS_FORWARDED
122 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
123
124 __ andi_(R11_tmp, R11_tmp, required_states);
125 __ beq(CR0, skip_prologue);
126
127 /* ==== Invoke runtime ==== */
128 // Save to-be-preserved registers.
129 int highest_preserve_register_index = 0;
130 {
131 if (preserve1 != noreg && preserve1->is_volatile()) {
132 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
133 }
134 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
135 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
136 }
137
138 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
139 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
140 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
141
142 __ save_LR(R11_tmp);
143 __ push_frame_reg_args(-BytesPerWord * highest_preserve_register_index,
144 R11_tmp);
145 }
146
147 // Invoke runtime.
148 address jrt_address = nullptr;
149 if (UseCompressedOops) {
150 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
151 } else {
152 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
153 }
154 assert(jrt_address != nullptr, "jrt routine cannot be found");
155
156 __ call_VM_leaf(jrt_address, src, dst, count);
157
158 // Restore to-be-preserved registers.
159 {
160 __ pop_frame();
161 __ restore_LR(R11_tmp);
162
163 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
164 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
165 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
166
167 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
168 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
169 }
170 if (preserve1 != noreg && preserve1->is_volatile()) {
171 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
172 }
173 }
174
175 __ bind(skip_prologue);
176 __ block_comment("} arraycopy_prologue (shenandoahgc)");
177 }
178
179 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
180 Register dst, Register count,
181 Register preserve) {
182 if (ShenandoahCardBarrier && is_reference_type(type)) {
183 __ block_comment("arraycopy_epilogue (shenandoahgc) {");
184 gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
185 __ block_comment("} arraycopy_epilogue (shenandoahgc)");
186 }
187 }
188
189 // The to-be-enqueued value can either be determined
190 // - dynamically by passing the reference's address information (load mode) or
191 // - statically by passing a register the value is stored in (preloaded mode)
192 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
193 // - for incremental-update barriers.
194 //
195 // decorators: The previous value's decorator set.
196 // In "load mode", the value must equal '0'.
197 // base: Base register of the reference's address (load mode).
198 // In "preloaded mode", the register must equal 'noreg'.
199 // ind_or_offs: Index or offset of the reference's address (load mode).
200 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
201 // pre_val: Register holding the to-be-stored value (preloaded mode).
202 // In "load mode", this register acts as a temporary register and must
203 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
204 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
205 void ShenandoahBarrierSetAssembler::satb_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
206 Register base, RegisterOrConstant ind_or_offs,
207 Register pre_val,
208 Register tmp1, Register tmp2,
209 MacroAssembler::PreservationLevel preservation_level) {
210 assert(ShenandoahSATBBarrier, "Should be checked by caller");
211 assert_different_registers(tmp1, tmp2, pre_val, noreg);
212
213 Label skip_barrier;
214
215 /* ==== Determine necessary runtime invocation preservation measures ==== */
216 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
217 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
218 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
219
220 // Check whether marking is active.
221 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
222
223 __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
224 __ beq(CR0, skip_barrier);
225
226 /* ==== Determine the reference's previous value ==== */
227 bool preloaded_mode = base == noreg;
228 Register pre_val_save = noreg;
229
230 if (preloaded_mode) {
231 // Previous value has been passed to the method, so it must not be determined manually.
232 // In case 'pre_val' is a volatile register, it must be saved across the C-call
233 // as callers may depend on its value.
234 // Unless the general purposes registers are saved anyway, one of the temporary registers
235 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
236 if (!preserve_gp_registers && pre_val->is_volatile()) {
237 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
238 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
239 }
240
241 if ((decorators & IS_NOT_NULL) != 0) {
242 #ifdef ASSERT
243 __ cmpdi(CR0, pre_val, 0);
244 __ asm_assert_ne("null oop is not allowed");
245 #endif // ASSERT
246 } else {
247 __ cmpdi(CR0, pre_val, 0);
248 __ beq(CR0, skip_barrier);
249 }
250 } else {
251 // Load from the reference address to determine the reference's current value (before the store is being performed).
252 // Contrary to the given value in "preloaded mode", it is not necessary to preserve it.
253 assert(decorators == 0, "decorator set must be empty");
254 assert(base != noreg, "base must be a register");
255 assert(!ind_or_offs.is_register() || ind_or_offs.as_register() != noreg, "ind_or_offs must be a register");
256 if (UseCompressedOops) {
257 __ lwz(pre_val, ind_or_offs, base);
258 } else {
259 __ ld(pre_val, ind_or_offs, base);
260 }
261
262 __ cmpdi(CR0, pre_val, 0);
263 __ beq(CR0, skip_barrier);
264
265 if (UseCompressedOops) {
266 __ decode_heap_oop_not_null(pre_val);
267 }
268 }
269
270 /* ==== Try to enqueue the to-be-stored value directly into thread's local SATB mark queue ==== */
271 {
272 Label runtime;
273 Register Rbuffer = tmp1, Rindex = tmp2;
274
275 // Check whether the queue has enough capacity to store another oop.
276 // If not, jump to the runtime to commit the buffer and to allocate a new one.
277 // (The buffer's index corresponds to the amount of remaining free space.)
278 __ ld(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
279 __ cmpdi(CR0, Rindex, 0);
280 __ beq(CR0, runtime); // If index == 0 (buffer is full), goto runtime.
281
282 // Capacity suffices. Decrement the queue's size by the size of one oop.
283 // (The buffer is filled contrary to the heap's growing direction, i.e., it is filled downwards.)
284 __ addi(Rindex, Rindex, -wordSize);
285 __ std(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
286
287 // Enqueue the previous value and skip the invocation of the runtime.
288 __ ld(Rbuffer, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
289 __ stdx(pre_val, Rbuffer, Rindex);
290 __ b(skip_barrier);
291
292 __ bind(runtime);
293 }
294
295 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
296 // Save to-be-preserved registers.
297 int nbytes_save = 0;
298
299 if (needs_frame) {
300 if (preserve_gp_registers) {
301 nbytes_save = (preserve_fp_registers
302 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
303 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
304 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
305 }
306
307 __ save_LR(tmp1);
308 __ push_frame_reg_args(nbytes_save, tmp2);
309 }
310
311 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
312 assert(pre_val_save != noreg, "nv_save must not be noreg");
313
314 // 'pre_val' register must be saved manually unless general-purpose are preserved in general.
315 __ mr(pre_val_save, pre_val);
316 }
317
318 // Invoke runtime.
319 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
320
321 // Restore to-be-preserved registers.
322 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
323 __ mr(pre_val, pre_val_save);
324 }
325
326 if (needs_frame) {
327 __ pop_frame();
328 __ restore_LR(tmp1);
329
330 if (preserve_gp_registers) {
331 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
332 }
333 }
334
335 __ bind(skip_barrier);
336 }
337
338 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler *masm, Register dst, Register tmp) {
339 __ block_comment("resolve_forward_pointer_not_null (shenandoahgc) {");
340
341 Register tmp1 = tmp,
342 R0_tmp2 = R0;
343 assert_different_registers(dst, tmp1, R0_tmp2, noreg);
344
345 // If the object has been evacuated, the mark word layout is as follows:
346 // | forwarding pointer (62-bit) | '11' (2-bit) |
347
348 // The invariant that stack/thread pointers have the lowest two bits cleared permits retrieving
349 // the forwarding pointer solely by inversing the lowest two bits.
350 // This invariant follows inevitably from hotspot's minimal alignment.
351 assert(markWord::marked_value <= (unsigned long) MinObjAlignmentInBytes,
352 "marked value must not be higher than hotspot's minimal alignment");
353
354 Label done;
355
356 // Load the object's mark word.
357 __ ld(tmp1, oopDesc::mark_offset_in_bytes(), dst);
358
359 // Load the bit mask for the lock bits.
360 __ li(R0_tmp2, markWord::lock_mask_in_place);
361
362 // Check whether all bits matching the bit mask are set.
363 // If that is the case, the object has been evacuated and the most significant bits form the forward pointer.
364 __ andc_(R0_tmp2, R0_tmp2, tmp1);
365
366 assert(markWord::lock_mask_in_place == markWord::marked_value,
367 "marked value must equal the value obtained when all lock bits are being set");
368 __ xori(tmp1, tmp1, markWord::lock_mask_in_place);
369 __ isel(dst, CR0, Assembler::equal, false, tmp1);
370
371 __ bind(done);
372 __ block_comment("} resolve_forward_pointer_not_null (shenandoahgc)");
373 }
374
375 // base: Base register of the reference's address.
376 // ind_or_offs: Index or offset of the reference's address (load mode).
377 // dst: Reference's address. In case the object has been evacuated, this is the to-space version
378 // of that object.
379 void ShenandoahBarrierSetAssembler::load_reference_barrier_impl(
380 MacroAssembler *masm, DecoratorSet decorators,
381 Register base, RegisterOrConstant ind_or_offs,
382 Register dst,
383 Register tmp1, Register tmp2,
384 MacroAssembler::PreservationLevel preservation_level) {
385 if (ind_or_offs.is_register()) {
386 assert_different_registers(tmp1, tmp2, base, ind_or_offs.as_register(), dst, noreg);
387 } else {
388 assert_different_registers(tmp1, tmp2, base, dst, noreg);
389 }
390
391 Label skip_barrier;
392
393 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
394 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
395 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
396 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
397 bool is_narrow = UseCompressedOops && !is_native;
398
399 /* ==== Check whether heap is stable ==== */
400 __ lbz(tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
401
402 if (is_strong) {
403 // For strong references, the heap is considered stable if "has forwarded" is not active.
404 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
405 __ beq(CR0, skip_barrier);
406 #ifdef ASSERT
407 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
408 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
409 __ asm_assert_ne("'has forwarded' is missing");
410 #endif // ASSERT
411 } else {
412 // For all non-strong references, the heap is considered stable if not any of "has forwarded",
413 // "root set processing", and "weak reference processing" is active.
414 // The additional phase conditions are in place to avoid the resurrection of weak references (see JDK-8266440).
415 Label skip_fastpath;
416 __ andi_(tmp1, tmp2, ShenandoahHeap::WEAK_ROOTS);
417 __ bne(CR0, skip_fastpath);
418
419 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
420 __ beq(CR0, skip_barrier);
421 #ifdef ASSERT
422 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
423 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
424 __ asm_assert_ne("'has forwarded' is missing");
425 #endif // ASSERT
426
427 __ bind(skip_fastpath);
428 }
429
430 /* ==== Check whether region is in collection set ==== */
431 if (is_strong) {
432 // Shenandoah stores metadata on regions in a continuous area of memory in which a single byte corresponds to
433 // an entire region of the shenandoah heap. At present, only the least significant bit is of significance
434 // and indicates whether the region is part of the collection set.
435 //
436 // All regions are of the same size and are always aligned by a power of two.
437 // Any address can thus be shifted by a fixed number of bits to retrieve the address prefix shared by
438 // all objects within that region (region identification bits).
439 //
440 // | unused bits | region identification bits | object identification bits |
441 // (Region size depends on a couple of criteria, such as page size, user-provided arguments and the max heap size.
442 // The number of object identification bits can thus not be determined at compile time.)
443 //
444 // ------------------------------------------------------- <--- cs (collection set) base address
445 // | lost space due to heap space base address -> 'ShenandoahHeap::in_cset_fast_test_addr()'
446 // | (region identification bits contain heap base offset)
447 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
448 // | collection set in the proper -> shift: 'region_size_bytes_shift_jint()'
449 // |
450 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
451 // + number of regions
452 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
453 __ srdi(tmp1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
454 __ lbzx(tmp2, tmp1, tmp2);
455 __ andi_(tmp2, tmp2, 1);
456 __ beq(CR0, skip_barrier);
457 }
458
459 /* ==== Invoke runtime ==== */
460 // Save to-be-preserved registers.
461 int nbytes_save = 0;
462
463 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
464 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
465 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
466
467 if (needs_frame) {
468 if (preserve_gp_registers) {
469 nbytes_save = (preserve_fp_registers
470 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
471 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
472 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
473 }
474
475 __ save_LR(tmp1);
476 __ push_frame_reg_args(nbytes_save, tmp1);
477 }
478
479 // Calculate the reference's absolute address.
480 __ add(R4_ARG2, ind_or_offs, base);
481
482 // Invoke runtime.
483 address jrt_address = nullptr;
484
485 if (is_strong) {
486 if (is_narrow) {
487 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
488 } else {
489 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
490 }
491 } else if (is_weak) {
492 if (is_narrow) {
493 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
494 } else {
495 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
496 }
497 } else {
498 assert(is_phantom, "only remaining strength");
499 assert(!is_narrow, "phantom access cannot be narrow");
500 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
501 }
502 assert(jrt_address != nullptr, "jrt routine cannot be found");
503
504 __ call_VM_leaf(jrt_address, dst /* reference */, R4_ARG2 /* reference address */);
505
506 // Restore to-be-preserved registers.
507 if (preserve_gp_registers) {
508 __ mr(R0, R3_RET);
509 } else {
510 __ mr_if_needed(dst, R3_RET);
511 }
512
513 if (needs_frame) {
514 __ pop_frame();
515 __ restore_LR(tmp1);
516
517 if (preserve_gp_registers) {
518 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
519 __ mr(dst, R0);
520 }
521 }
522
523 __ bind(skip_barrier);
524 }
525
526 // base: Base register of the reference's address.
527 // ind_or_offs: Index or offset of the reference's address.
528 // L_handle_null: An optional label that will be jumped to if the reference is null.
529 void ShenandoahBarrierSetAssembler::load_at(
530 MacroAssembler *masm, DecoratorSet decorators, BasicType type,
531 Register base, RegisterOrConstant ind_or_offs, Register dst,
532 Register tmp1, Register tmp2,
533 MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
534 // Register must not clash, except 'base' and 'dst'.
535 if (ind_or_offs.is_register()) {
536 if (base != noreg) {
537 assert_different_registers(tmp1, tmp2, base, ind_or_offs.register_or_noreg(), R0, noreg);
538 }
539 assert_different_registers(tmp1, tmp2, dst, ind_or_offs.register_or_noreg(), R0, noreg);
540 } else {
541 if (base == noreg) {
542 assert_different_registers(tmp1, tmp2, base, R0, noreg);
543 }
544 assert_different_registers(tmp1, tmp2, dst, R0, noreg);
545 }
546
547 /* ==== Apply load barrier, if required ==== */
548 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
549 assert(is_reference_type(type), "need_load_reference_barrier must check whether type is a reference type");
550
551 // If 'dst' clashes with either 'base' or 'ind_or_offs', use an intermediate result register
552 // to keep the values of those alive until the load reference barrier is applied.
553 Register intermediate_dst = (dst == base || (ind_or_offs.is_register() && dst == ind_or_offs.as_register()))
554 ? tmp2
555 : dst;
556
557 BarrierSetAssembler::load_at(masm, decorators, type,
558 base, ind_or_offs,
559 intermediate_dst,
560 tmp1, noreg,
561 preservation_level, L_handle_null);
562
563 load_reference_barrier(masm, decorators,
564 base, ind_or_offs,
565 intermediate_dst,
566 tmp1, R0,
567 preservation_level);
568
569 __ mr_if_needed(dst, intermediate_dst);
570 } else {
571 BarrierSetAssembler::load_at(masm, decorators, type,
572 base, ind_or_offs,
573 dst,
574 tmp1, tmp2,
575 preservation_level, L_handle_null);
576 }
577
578 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
579 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
580 if (ShenandoahSATBBarrier) {
581 __ block_comment("keep_alive_barrier (shenandoahgc) {");
582 satb_barrier_impl(masm, 0, noreg, noreg, dst, tmp1, tmp2, preservation_level);
583 __ block_comment("} keep_alive_barrier (shenandoahgc)");
584 }
585 }
586 }
587
588 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
589 assert(ShenandoahCardBarrier, "Should have been checked by caller");
590 assert_different_registers(base, tmp, R0);
591
592 if (ind_or_offs.is_constant()) {
593 __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
594 } else {
595 __ add(base, ind_or_offs.as_register(), base);
596 }
597
598 __ ld(tmp, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread); /* tmp = *[R16_thread + card_table_offset] */
599 __ srdi(base, base, CardTable::card_shift());
600 __ li(R0, CardTable::dirty_card_val());
601 __ stbx(R0, tmp, base);
602 }
603
604 // base: Base register of the reference's address.
605 // ind_or_offs: Index or offset of the reference's address.
606 // val: To-be-stored value/reference's new value.
607 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
608 Register base, RegisterOrConstant ind_or_offs, Register val,
609 Register tmp1, Register tmp2, Register tmp3,
610 MacroAssembler::PreservationLevel preservation_level) {
611 // 1: non-reference types require no barriers
612 if (!is_reference_type(type)) {
613 BarrierSetAssembler::store_at(masm, decorators, type,
614 base, ind_or_offs,
615 val,
616 tmp1, tmp2, tmp3,
617 preservation_level);
618 return;
619 }
620
621 bool storing_non_null = (val != noreg);
622
623 // 2: pre-barrier: SATB needs the previous value
624 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
625 satb_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
626 }
627
628 // Store!
629 BarrierSetAssembler::store_at(masm, decorators, type,
630 base, ind_or_offs,
631 val,
632 tmp1, tmp2, tmp3,
633 preservation_level);
634
635 // 3: post-barrier: card barrier needs store address
636 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
637 card_barrier(masm, base, ind_or_offs, tmp1);
638 }
639 }
640
641 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
642 Register dst, Register jni_env, Register obj,
643 Register tmp, Label &slowpath) {
644 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
645
646 assert_different_registers(jni_env, obj, tmp);
647
648 Label done;
649
650 // Fast path: Reference is null (JNI tags are zero for null pointers).
651 __ cmpdi(CR0, obj, 0);
652 __ beq(CR0, done);
653
654 // Resolve jobject using standard implementation.
655 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
656
657 // Check whether heap is stable.
658 __ lbz(tmp,
659 in_bytes(ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()),
660 jni_env);
661
662 __ andi_(tmp, tmp, ShenandoahHeap::EVACUATION | ShenandoahHeap::HAS_FORWARDED);
663 __ bne(CR0, slowpath);
664
665 __ bind(done);
666 __ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
667 }
668
669 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
670 Register obj, Register tmp, Label &slow_path) {
671 __ block_comment("try_peek_weak_handle_in_nmethod (shenandoahgc) {");
672
673 assert_different_registers(weak_handle, tmp, noreg);
674 assert_different_registers(obj, tmp, noreg);
675
676
677 Label done;
678
679 // Peek weak handle using the standard implementation.
680 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
681
682 // Check if the reference is null, and if it is, take the fast path.
683 __ cmpdi(CR0, obj, 0);
684 __ beq(CR0, done);
685
686 // Check if the heap is under weak-reference/roots processing, in
687 // which case we need to take the slow path.
688 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
689 __ andi_(tmp, tmp, ShenandoahHeap::WEAK_ROOTS);
690 __ bne(CR0, slow_path);
691 __ bind(done);
692
693 __ block_comment("} try_peek_weak_handle_in_nmethod (shenandoahgc)");
694 }
695
696 // Special shenandoah CAS implementation that handles false negatives due
697 // to concurrent evacuation. That is, the CAS operation is intended to succeed in
698 // the following scenarios (success criteria):
699 // s1) The reference pointer ('base_addr') equals the expected ('expected') pointer.
700 // s2) The reference pointer refers to the from-space version of an already-evacuated
701 // object, whereas the expected pointer refers to the to-space version of the same object.
702 // Situations in which the reference pointer refers to the to-space version of an object
703 // and the expected pointer refers to the from-space version of the same object can not occur due to
704 // shenandoah's strong to-space invariant. This also implies that the reference stored in 'new_val'
705 // can not refer to the from-space version of an already-evacuated object.
706 //
707 // To guarantee correct behavior in concurrent environments, two races must be addressed:
708 // r1) A concurrent thread may heal the reference pointer (i.e., it is no longer referring to the
709 // from-space version but to the to-space version of the object in question).
710 // In this case, the CAS operation should succeed.
711 // r2) A concurrent thread may mutate the reference (i.e., the reference pointer refers to an entirely different object).
712 // In this case, the CAS operation should fail.
713 //
714 // By default, the value held in the 'result' register is zero to indicate failure of CAS,
715 // non-zero to indicate success. If 'is_cae' is set, the result is the most recently fetched
716 // value from 'base_addr' rather than a boolean success indicator.
717 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler *masm, Register base_addr,
718 Register expected, Register new_val, Register tmp1, Register tmp2,
719 bool is_cae, Register result) {
720 __ block_comment("cmpxchg_oop (shenandoahgc) {");
721
722 assert_different_registers(base_addr, new_val, tmp1, tmp2, result, R0);
723 assert_different_registers(base_addr, expected, tmp1, tmp2, result, R0);
724
725 // Potential clash of 'success_flag' and 'tmp' is being accounted for.
726 Register success_flag = is_cae ? noreg : result,
727 current_value = is_cae ? result : tmp1,
728 tmp = is_cae ? tmp1 : result,
729 initial_value = tmp2;
730
731 Label done, step_four;
732
733 __ bind(step_four);
734
735 /* ==== Step 1 ("Standard" CAS) ==== */
736 // Fast path: The values stored in 'expected' and 'base_addr' are equal.
737 // Given that 'expected' must refer to the to-space object of an evacuated object (strong to-space invariant),
738 // no special processing is required.
739 if (UseCompressedOops) {
740 __ cmpxchgw(CR0, current_value, expected, new_val, base_addr, MacroAssembler::MemBarNone,
741 false, success_flag, nullptr, true);
742 } else {
743 __ cmpxchgd(CR0, current_value, expected, new_val, base_addr, MacroAssembler::MemBarNone,
744 false, success_flag, nullptr, true);
745 }
746
747 // Skip the rest of the barrier if the CAS operation succeeds immediately.
748 // If it does not, the value stored at the address is either the from-space pointer of the
749 // referenced object (success criteria s2)) or simply another object.
750 __ beq(CR0, done);
751
752 /* ==== Step 2 (Null check) ==== */
753 // The success criteria s2) cannot be matched with a null pointer
754 // (null pointers cannot be subject to concurrent evacuation). The failure of the CAS operation is thus legitimate.
755 __ cmpdi(CR0, current_value, 0);
756 __ beq(CR0, done);
757
758 /* ==== Step 3 (reference pointer refers to from-space version; success criteria s2)) ==== */
759 // To check whether the reference pointer refers to the from-space version, the forward
760 // pointer of the object referred to by the reference is resolved and compared against the expected pointer.
761 // If this check succeed, another CAS operation is issued with the from-space pointer being the expected pointer.
762 //
763 // Save the potential from-space pointer.
764 __ mr(initial_value, current_value);
765
766 // Resolve forward pointer.
767 if (UseCompressedOops) { __ decode_heap_oop_not_null(current_value); }
768 resolve_forward_pointer_not_null(masm, current_value, tmp);
769 if (UseCompressedOops) { __ encode_heap_oop_not_null(current_value); }
770
771 if (!is_cae) {
772 // 'success_flag' was overwritten by call to 'resovle_forward_pointer_not_null'.
773 // Load zero into register for the potential failure case.
774 __ li(success_flag, 0);
775 }
776 __ cmpd(CR0, current_value, expected);
777 __ bne(CR0, done);
778
779 // Discard fetched value as it might be a reference to the from-space version of an object.
780 if (UseCompressedOops) {
781 __ cmpxchgw(CR0, R0, initial_value, new_val, base_addr, MacroAssembler::MemBarNone,
782 false, success_flag);
783 } else {
784 __ cmpxchgd(CR0, R0, initial_value, new_val, base_addr, MacroAssembler::MemBarNone,
785 false, success_flag);
786 }
787
788 /* ==== Step 4 (Retry CAS with to-space pointer (success criteria s2) under race r1)) ==== */
789 // The reference pointer could have been healed whilst the previous CAS operation was being performed.
790 // Another CAS operation must thus be issued with the to-space pointer being the expected pointer.
791 // If that CAS operation fails as well, race r2) must have occurred, indicating that
792 // the operation failure is legitimate.
793 //
794 // To keep the code's size small and thus improving cache (icache) performance, this highly
795 // unlikely case should be handled by the smallest possible code. Instead of emitting a third,
796 // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
797 // (passed arguments are identical).
798 //
799 // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
800 // to fail. Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
801 // It is thus important to ensure that a re-execution of those steps does not put program correctness
802 // at risk:
803 // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
804 // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
805 // fails. Unless the reference has been updated in the meanwhile once again, this is
806 // guaranteed to be the case.
807 // In case of a concurrent update, the CAS would be retried again. This is legitimate
808 // in terms of program correctness (even though it is not desired).
809 __ bne(CR0, step_four);
810
811 __ bind(done);
812 __ block_comment("} cmpxchg_oop (shenandoahgc)");
813 }
814
815 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
816 Register addr, Register count, Register preserve) {
817 assert(ShenandoahCardBarrier, "Should have been checked by caller");
818 assert_different_registers(addr, count, R0);
819
820 Label L_skip_loop, L_store_loop;
821
822 __ sldi_(count, count, LogBytesPerHeapOop);
823
824 // Zero length? Skip.
825 __ beq(CR0, L_skip_loop);
826
827 __ addi(count, count, -BytesPerHeapOop);
828 __ add(count, addr, count);
829 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
830 __ srdi(addr, addr, CardTable::card_shift());
831 __ srdi(count, count, CardTable::card_shift());
832 __ subf(count, addr, count);
833 __ ld(R0, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
834 __ add(addr, addr, R0);
835 __ addi(count, count, 1);
836 __ li(R0, 0);
837 __ mtctr(count);
838
839 // Byte store loop
840 __ bind(L_store_loop);
841 __ stb(R0, 0, addr);
842 __ addi(addr, addr, 1);
843 __ bdnz(L_store_loop);
844 __ bind(L_skip_loop);
845 }
846
847 #undef __
848
849 #ifdef COMPILER1
850
851 #define __ ce->masm()->
852
853 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
854 __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
855
856 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
857 __ bind(*stub->entry());
858
859 // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
860 // This stub is the slowpath of that function.
861
862 assert(stub->pre_val()->is_register(), "pre_val must be a register");
863 Register pre_val = stub->pre_val()->as_register();
864
865 // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
866 // ("preloaded mode" of the store barrier).
867 if (stub->do_load()) {
868 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false);
869 }
870
871 // Fast path: Reference is null.
872 __ cmpdi(CR0, pre_val, 0);
873 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
874
875 // Argument passing via the stack.
876 __ std(pre_val, -8, R1_SP);
877
878 __ load_const_optimized(R0, bs->pre_barrier_c1_runtime_code_blob()->code_begin());
879 __ call_stub(R0);
880
881 __ b(*stub->continuation());
882 __ block_comment("} gen_pre_barrier_stub (shenandoahgc)");
883 }
884
885 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler *ce,
886 ShenandoahLoadReferenceBarrierStub *stub) {
887 __ block_comment("gen_load_reference_barrier_stub (shenandoahgc) {");
888
889 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
890 __ bind(*stub->entry());
891
892 Register obj = stub->obj()->as_register();
893 Register res = stub->result()->as_register();
894 Register addr = stub->addr()->as_pointer_register();
895 Register tmp1 = stub->tmp1()->as_register();
896 Register tmp2 = stub->tmp2()->as_register();
897 assert_different_registers(addr, res, tmp1, tmp2);
898
899 assert(R3_RET == res, "res must be r3");
900
901 if (res != obj) {
902 __ mr(res, obj);
903 }
904
905 DecoratorSet decorators = stub->decorators();
906
907 /* ==== Check whether region is in collection set ==== */
908 // GC status (unstable) has already been verified by 'ShenandoahBarrierSetC1::load_reference_barrier_impl'.
909 // This stub is the slowpath of that function.
910
911 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
912 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
913 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
914 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
915
916 if (is_strong) {
917 // Check whether object is in collection set.
918 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
919 __ srdi(tmp1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
920 __ lbzx(tmp2, tmp1, tmp2);
921
922 __ andi_(tmp2, tmp2, 1);
923 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
924 }
925
926 address blob_addr = nullptr;
927
928 if (is_strong) {
929 if (is_native) {
930 blob_addr = bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin();
931 } else {
932 blob_addr = bs->load_reference_barrier_strong_rt_code_blob()->code_begin();
933 }
934 } else if (is_weak) {
935 blob_addr = bs->load_reference_barrier_weak_rt_code_blob()->code_begin();
936 } else {
937 assert(is_phantom, "only remaining strength");
938 blob_addr = bs->load_reference_barrier_phantom_rt_code_blob()->code_begin();
939 }
940
941 assert(blob_addr != nullptr, "code blob cannot be found");
942
943 // Argument passing via the stack. 'obj' is passed implicitly (as asserted above).
944 __ std(addr, -8, R1_SP);
945
946 __ load_const_optimized(tmp1, blob_addr, tmp2);
947 __ call_stub(tmp1);
948
949 // 'res' is 'R3_RET'. The result is thus already in the correct register.
950
951 __ b(*stub->continuation());
952 __ block_comment("} gen_load_reference_barrier_stub (shenandoahgc)");
953 }
954
955 #undef __
956
957 #define __ sasm->
958
959 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler *sasm) {
960 __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
961
962 Label runtime, skip_barrier;
963 BarrierSet *bs = BarrierSet::barrier_set();
964
965 // Argument passing via the stack.
966 const int caller_stack_slots = 3;
967
968 Register R0_pre_val = R0;
969 __ ld(R0, -8, R1_SP);
970 Register R11_tmp1 = R11_scratch1;
971 __ std(R11_tmp1, -16, R1_SP);
972 Register R12_tmp2 = R12_scratch2;
973 __ std(R12_tmp2, -24, R1_SP);
974
975 /* ==== Check whether marking is active ==== */
976 // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
977 // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
978 __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
979
980 __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::MARKING);
981 __ beq(CR0, skip_barrier);
982
983 /* ==== Add previous value directly to thread-local SATB mark queue ==== */
984 // Check queue's capacity. Jump to runtime if no free slot is available.
985 __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
986 __ cmpdi(CR0, R12_tmp2, 0);
987 __ beq(CR0, runtime);
988
989 // Capacity suffices. Decrement the queue's size by one slot (size of one oop).
990 __ addi(R12_tmp2, R12_tmp2, -wordSize);
991 __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
992
993 // Enqueue the previous value and skip the runtime invocation.
994 __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
995 __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
996 __ b(skip_barrier);
997
998 __ bind(runtime);
999
1000 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
1001 // Save to-be-preserved registers.
1002 const int nbytes_save = (MacroAssembler::num_volatile_regs + caller_stack_slots) * BytesPerWord;
1003 __ save_volatile_gprs(R1_SP, -nbytes_save);
1004 __ save_LR(R11_tmp1);
1005 __ push_frame_reg_args(nbytes_save, R11_tmp1);
1006
1007 // Invoke runtime.
1008 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), R0_pre_val);
1009
1010 // Restore to-be-preserved registers.
1011 __ pop_frame();
1012 __ restore_LR(R11_tmp1);
1013 __ restore_volatile_gprs(R1_SP, -nbytes_save);
1014
1015 __ bind(skip_barrier);
1016
1017 // Restore spilled registers.
1018 __ ld(R11_tmp1, -16, R1_SP);
1019 __ ld(R12_tmp2, -24, R1_SP);
1020
1021 __ blr();
1022 __ block_comment("} generate_c1_pre_barrier_runtime_stub (shenandoahgc)");
1023 }
1024
1025 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler *sasm,
1026 DecoratorSet decorators) {
1027 __ block_comment("generate_c1_load_reference_barrier_runtime_stub (shenandoahgc) {");
1028
1029 // Argument passing via the stack.
1030 const int caller_stack_slots = 1;
1031
1032 // Save to-be-preserved registers.
1033 const int nbytes_save = (MacroAssembler::num_volatile_regs - 1 // 'R3_ARG1' is skipped
1034 + caller_stack_slots) * BytesPerWord;
1035 __ save_volatile_gprs(R1_SP, -nbytes_save, true, false);
1036
1037 // Load arguments from stack.
1038 // No load required, as caller has already loaded obj into R3.
1039 Register R3_obj = R3_ARG1;
1040 Register R4_load_addr = R4_ARG2;
1041 __ ld(R4_load_addr, -8, R1_SP);
1042
1043 Register R11_tmp = R11_scratch1;
1044
1045 /* ==== Invoke runtime ==== */
1046 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
1047 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
1048 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
1049 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
1050
1051 address jrt_address = nullptr;
1052
1053 if (is_strong) {
1054 if (is_native) {
1055 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1056 } else {
1057 if (UseCompressedOops) {
1058 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
1059 } else {
1060 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1061 }
1062 }
1063 } else if (is_weak) {
1064 assert(!is_native, "weak load reference barrier must not be called off-heap");
1065 if (UseCompressedOops) {
1066 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
1067 } else {
1068 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
1069 }
1070 } else {
1071 assert(is_phantom, "reference type must be phantom");
1072 assert(is_native, "phantom load reference barrier must be called off-heap");
1073 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
1074 }
1075 assert(jrt_address != nullptr, "load reference barrier runtime routine cannot be found");
1076
1077 __ save_LR(R11_tmp);
1078 __ push_frame_reg_args(nbytes_save, R11_tmp);
1079
1080 // Invoke runtime. Arguments are already stored in the corresponding registers.
1081 __ call_VM_leaf(jrt_address, R3_obj, R4_load_addr);
1082
1083 // Restore to-be-preserved registers.
1084 __ pop_frame();
1085 __ restore_LR(R11_tmp);
1086 __ restore_volatile_gprs(R1_SP, -nbytes_save, true, false); // Skip 'R3_RET' register.
1087
1088 __ blr();
1089 __ block_comment("} generate_c1_load_reference_barrier_runtime_stub (shenandoahgc)");
1090 }
1091
1092 #undef __
1093
1094 #endif // COMPILER1
1095
1096 #ifdef COMPILER2
1097
1098 #undef __
1099 #define __ masm->
1100
1101 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Register addr, int disp, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
1102 if (is_narrow) {
1103 __ lwz(dst, disp, addr);
1104 } else {
1105 __ ld(dst, disp, addr);
1106 }
1107 if (is_acquire) {
1108 __ twi_0(dst);
1109 __ isync();
1110 }
1111
1112 ShenandoahBarrierStubC2::load_post(masm, node, dst, Address(addr, disp), tmp1, tmp2, is_narrow);
1113 }
1114
1115 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
1116 Register dst, int disp, bool dst_narrow, Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
1117
1118 ShenandoahBarrierStubC2::store_pre(masm, node, tmp1, Address(dst, disp), tmp2, tmp3, dst_narrow);
1119
1120 if (dst_narrow && !src_narrow) {
1121 // Need to encode into tmp, because we cannot clobber src.
1122 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
1123 src = __ encode_heap_oop(tmp1, src);
1124 } else {
1125 src = __ encode_heap_oop_not_null(tmp1, src);
1126 }
1127 }
1128 if (dst_narrow) {
1129 __ stw(src, disp, dst);
1130 } else {
1131 __ std(src, disp, dst);
1132 }
1133
1134 ShenandoahBarrierStubC2::store_post(masm, node, Address(dst, disp), tmp1, tmp2);
1135 }
1136
1137 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr, Register oldval,
1138 Register newval, Register tmp1, Register tmp2, Register tmp3, bool exchange, bool narrow, bool weak, bool acquire) {
1139
1140 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp1, addr, tmp2, tmp3, narrow);
1141
1142 Register dest_current = exchange ? res : R0;
1143 Register int_flag = exchange ? noreg : res;
1144 int semantics = MacroAssembler::MemBarNone;
1145
1146 if (acquire) {
1147 semantics = support_IRIW_for_not_multiple_copy_atomic_cpu ?
1148 MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter;
1149 }
1150
1151 if (narrow) {
1152 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
1153 __ cmpxchgw(CR0, dest_current, oldval, newval, addr,
1154 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
1155 int_flag, nullptr, true, weak);
1156 } else {
1157 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
1158 __ cmpxchgd(CR0, dest_current, oldval, newval, addr,
1159 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
1160 int_flag, nullptr, true, weak);
1161 }
1162
1163 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
1164 }
1165
1166 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval, Register newval, Register addr, Register tmp1, Register tmp2, Register tmp3) {
1167 bool is_narrow = node->bottom_type()->isa_narrowoop();
1168
1169 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp1, addr, tmp2, tmp3, is_narrow);
1170
1171 if (is_narrow) {
1172 __ getandsetw(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
1173 } else {
1174 __ getandsetd(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
1175 }
1176
1177 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
1178 __ isync();
1179 } else {
1180 __ sync();
1181 }
1182
1183 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
1184 }
1185
1186 #undef __
1187 #define __ masm.
1188
1189 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
1190 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1191 assert_different_registers(tmp1, tmp2, address.index(), address.base());
1192
1193 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
1194 if (address.index() == noreg) {
1195 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
1196 } else {
1197 __ add(tmp2, address.index(), address.base());
1198 if (address.disp() != 0) {
1199 __ addi(tmp2, tmp2, address.disp());
1200 }
1201 }
1202 __ srdi(tmp2, tmp2, CardTable::card_shift());
1203 __ li(R0, CardTable::dirty_card_val());
1204 __ stbx(R0, tmp2, tmp1);
1205 }
1206
1207 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
1208 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1209
1210 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)), R16_thread);
1211 __ cmpdi(CR0, tmp, 0);
1212 // Branch to entry if not equal
1213 __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *entry());
1214 // This is were the slowpath stub will return to
1215 __ bind(*continuation());
1216 }
1217
1218 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1219 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1220 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1221
1222 __ bind(*entry());
1223
1224 // If we need to load ourselves, do it here.
1225 if (_do_load) {
1226 if (_narrow) {
1227 __ lwz(_obj, _addr.disp(), _addr.base());
1228 } else {
1229 __ ld(_obj, _addr.disp(), _addr.base());
1230 }
1231 }
1232
1233 // If the object is null, there is no point in applying barriers.
1234 maybe_far_jump_if_zero(masm, _obj);
1235
1236 // We need to make sure that loads done by callers survive across slow-path calls.
1237 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1238 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
1239 if (!_do_load || needs_both_barriers) {
1240 preserve(_obj);
1241 }
1242
1243 // Go for barriers. Barriers can return straight to continuation, as long
1244 // as another barrier is not needed and we can reach the fastpath.
1245 if (needs_both_barriers) {
1246 keepalive(masm, nullptr);
1247 lrb(masm);
1248 } else if (_needs_keep_alive_barrier) {
1249 keepalive(masm, continuation());
1250 } else if (_needs_load_ref_barrier) {
1251 lrb(masm);
1252 } else {
1253 ShouldNotReachHere();
1254 }
1255 }
1256
1257 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1258 __ cmpdi(CR0, reg, 0);
1259 // Branch to continuation if equal
1260 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
1261 }
1262
1263 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1264 const int gcstate_offset = in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING));
1265 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1266 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1267 Label L_through, L_slowpath;
1268
1269 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1270 if (_needs_load_ref_barrier) {
1271 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
1272 __ lbz(_tmp1, gcstate_offset, R16_thread);
1273 __ cmpdi(CR0, _tmp1, 0);
1274 __ beq(CR0, L_through);
1275 }
1276
1277 // Fast-path: put object into buffer.
1278 // If buffer is already full, go slow.
1279 __ ld(_tmp1, index_offset, R16_thread);
1280 __ cmpdi(CR0, _tmp1, 0);
1281 __ beq(CR0, L_slowpath);
1282 __ addi(_tmp1, _tmp1, -wordSize);
1283 __ std(_tmp1, index_offset, R16_thread);
1284 __ ld(_tmp2, buffer_offset, R16_thread);
1285
1286 // Store the object in queue.
1287 // If object is narrow, we need to decode it before inserting.
1288 if (_narrow) {
1289 __ add(_tmp2, _tmp2, _tmp1);
1290 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1291 __ stdx(decoded, _tmp2);
1292 } else {
1293 __ stdx(_obj, _tmp2, _tmp1);
1294 }
1295
1296 // Fast-path exits here.
1297 if (L_done != nullptr) {
1298 __ b(*L_done);
1299 } else {
1300 __ b(L_through);
1301 }
1302
1303 // Slow-path: call runtime to handle.
1304 __ bind(L_slowpath);
1305
1306 {
1307 SaveLiveRegisters slr(&masm, this);
1308
1309 // Go to runtime and handle the rest there.
1310 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1311 }
1312
1313 if (L_done != nullptr) {
1314 __ b(*L_done);
1315 } else {
1316 __ bind(L_through);
1317 }
1318 }
1319
1320 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1321 Label L_slow;
1322
1323 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1324 if (_needs_keep_alive_barrier) {
1325 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1326 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)), R16_thread);
1327 maybe_far_jump_if_zero(masm, _tmp1);
1328 }
1329
1330 // If weak references are being processed, weak/phantom loads need to go slow,
1331 // regardless of their cset status.
1332 if (_needs_load_ref_weak_barrier) {
1333 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)), R16_thread);
1334 __ cmpdi(CR0, _tmp1, 0);
1335 __ bne(CR0, L_slow);
1336 }
1337
1338 // Cset-check. Fall-through to slow if in collection set.
1339 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1340 if (_narrow) {
1341 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1342 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1343 } else {
1344 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1345 }
1346 __ lbzx(_tmp2, _tmp2, _tmp1);
1347 maybe_far_jump_if_zero(masm, _tmp2);
1348
1349 // Slow path
1350 __ bind(L_slow);
1351
1352 // Obj is the result, need to temporarily stop preserving it.
1353 bool is_obj_preserved = is_preserved(_obj);
1354 if (is_obj_preserved) {
1355 dont_preserve(_obj);
1356 }
1357 {
1358 SaveLiveRegisters slr(&masm, this);
1359
1360 // Shuffle in the arguments. The end result should be:
1361 // c_rarg0 <-- obj
1362 // c_rarg1 <-- lea(addr)
1363 Register c_rarg0 = R3_ARG1;
1364 Register c_rarg1 = R4_ARG2;
1365 if (c_rarg0 == _obj) {
1366 __ addi(c_rarg1, _addr.base(), _addr.disp());
1367 } else if (c_rarg1 == _obj) {
1368 // Set up arguments in reverse, and then flip them
1369 __ addi(c_rarg0, _addr.base(), _addr.disp());
1370 // flip them
1371 __ mr(_tmp1, c_rarg0);
1372 __ mr(c_rarg0, c_rarg1);
1373 __ mr(c_rarg1, _tmp1);
1374 } else {
1375 assert_different_registers(c_rarg1, _obj);
1376 __ addi(c_rarg1, _addr.base(), _addr.disp());
1377 __ mr(c_rarg0, _obj);
1378 }
1379
1380 // Go to runtime and handle the rest there.
1381 __ call_VM_leaf(lrb_runtime_entry_addr(), c_rarg0, c_rarg1);
1382
1383 // Save the result where needed.
1384 if (_obj != R3_RET) {
1385 __ mr(_obj, R3_RET);
1386 }
1387 }
1388 if (is_obj_preserved) {
1389 preserve(_obj);
1390 }
1391
1392 __ b(*continuation());
1393 }
1394
1395 int ShenandoahBarrierStubC2::available_gp_registers() {
1396 Unimplemented(); // Not used
1397 return 0;
1398 }
1399
1400 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1401 Unimplemented(); // Not used
1402 return true;
1403 }
1404
1405 void ShenandoahBarrierStubC2::post_init() {
1406 // Do nothing.
1407 }
1408
1409 #endif // COMPILER2