1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2025, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahForwarding.hpp"
35 #include "gc/shenandoah/shenandoahHeap.hpp"
36 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
37 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "macroAssembler_ppc.hpp"
42 #include "runtime/javaThread.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/globalDefinitions.hpp"
45 #include "vm_version_ppc.hpp"
46 #ifdef COMPILER1
47 #include "c1/c1_LIRAssembler.hpp"
48 #include "c1/c1_MacroAssembler.hpp"
49 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
50 #endif
51
52 #define __ masm->
53
54 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
55 Register base, RegisterOrConstant ind_or_offs,
56 Register tmp1, Register tmp2, Register tmp3,
57 MacroAssembler::PreservationLevel preservation_level) {
58 if (ShenandoahSATBBarrier) {
59 __ block_comment("satb_barrier (shenandoahgc) {");
60 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
61 __ block_comment("} satb_barrier (shenandoahgc)");
62 }
63 }
64
65 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
66 Register base, RegisterOrConstant ind_or_offs,
67 Register dst,
68 Register tmp1, Register tmp2,
69 MacroAssembler::PreservationLevel preservation_level) {
70 if (ShenandoahLoadRefBarrier) {
71 __ block_comment("load_reference_barrier (shenandoahgc) {");
72 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
73 __ block_comment("} load_reference_barrier (shenandoahgc)");
74 }
75 }
76
77 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
78 Register src, Register dst, Register count,
79 Register preserve1, Register preserve2) {
80 Register R11_tmp = R11_scratch1;
81
82 assert_different_registers(src, dst, count, R11_tmp, noreg);
83 if (preserve1 != noreg) {
84 // Technically not required, but likely to indicate an error.
85 assert_different_registers(preserve1, preserve2);
86 }
87
88 /* ==== Check whether barrier is required (optimizations) ==== */
89 // Fast path: Component type of array is not a reference type.
90 if (!is_reference_type(type)) {
91 return;
92 }
93
94 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
95
96 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
97 // any useful information.
98 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahLoadRefBarrier) {
99 return;
100 }
101
102 __ block_comment("arraycopy_prologue (shenandoahgc) {");
103 Label skip_prologue;
104
105 // Fast path: Array is of length zero.
106 __ cmpdi(CR0, count, 0);
107 __ beq(CR0, skip_prologue);
108
109 /* ==== Check whether barrier is required (gc state) ==== */
110 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
111 R16_thread);
112
113 // The set of garbage collection states requiring barriers depends on the available barrier types and the
114 // type of the reference in question.
115 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
116 // for the garbage collector.
117 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
118 ? ShenandoahHeap::HAS_FORWARDED
119 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
120
121 __ andi_(R11_tmp, R11_tmp, required_states);
122 __ beq(CR0, skip_prologue);
123
124 /* ==== Invoke runtime ==== */
125 // Save to-be-preserved registers.
126 int highest_preserve_register_index = 0;
127 {
128 if (preserve1 != noreg && preserve1->is_volatile()) {
129 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
130 }
131 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
132 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
133 }
134
135 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
136 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
137 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
138
139 __ save_LR(R11_tmp);
140 __ push_frame_reg_args(-BytesPerWord * highest_preserve_register_index,
141 R11_tmp);
142 }
143
144 // Invoke runtime.
145 address jrt_address = nullptr;
146 if (UseCompressedOops) {
147 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
148 } else {
149 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
150 }
151 assert(jrt_address != nullptr, "jrt routine cannot be found");
152
153 __ call_VM_leaf(jrt_address, src, dst, count);
154
155 // Restore to-be-preserved registers.
156 {
157 __ pop_frame();
158 __ restore_LR(R11_tmp);
159
160 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
161 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
162 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
163
164 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
165 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
166 }
167 if (preserve1 != noreg && preserve1->is_volatile()) {
168 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
169 }
170 }
171
172 __ bind(skip_prologue);
173 __ block_comment("} arraycopy_prologue (shenandoahgc)");
174 }
175
176 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
177 Register dst, Register count,
178 Register preserve) {
179 if (ShenandoahCardBarrier && is_reference_type(type)) {
180 __ block_comment("arraycopy_epilogue (shenandoahgc) {");
181 gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
182 __ block_comment("} arraycopy_epilogue (shenandoahgc)");
183 }
184 }
185
186 // The to-be-enqueued value can either be determined
187 // - dynamically by passing the reference's address information (load mode) or
188 // - statically by passing a register the value is stored in (preloaded mode)
189 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
190 // - for incremental-update barriers.
191 //
192 // decorators: The previous value's decorator set.
193 // In "load mode", the value must equal '0'.
194 // base: Base register of the reference's address (load mode).
195 // In "preloaded mode", the register must equal 'noreg'.
196 // ind_or_offs: Index or offset of the reference's address (load mode).
197 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
198 // pre_val: Register holding the to-be-stored value (preloaded mode).
199 // In "load mode", this register acts as a temporary register and must
200 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
201 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
202 void ShenandoahBarrierSetAssembler::satb_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
203 Register base, RegisterOrConstant ind_or_offs,
204 Register pre_val,
205 Register tmp1, Register tmp2,
206 MacroAssembler::PreservationLevel preservation_level) {
207 assert(ShenandoahSATBBarrier, "Should be checked by caller");
208 assert_different_registers(tmp1, tmp2, pre_val, noreg);
209
210 Label skip_barrier;
211
212 /* ==== Determine necessary runtime invocation preservation measures ==== */
213 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
214 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
215 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
216
217 // Check whether marking is active.
218 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
219
220 __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
221 __ beq(CR0, skip_barrier);
222
223 /* ==== Determine the reference's previous value ==== */
224 bool preloaded_mode = base == noreg;
225 Register pre_val_save = noreg;
226
227 if (preloaded_mode) {
228 // Previous value has been passed to the method, so it must not be determined manually.
229 // In case 'pre_val' is a volatile register, it must be saved across the C-call
230 // as callers may depend on its value.
231 // Unless the general purposes registers are saved anyway, one of the temporary registers
232 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
233 if (!preserve_gp_registers && pre_val->is_volatile()) {
234 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
235 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
236 }
237
238 if ((decorators & IS_NOT_NULL) != 0) {
239 #ifdef ASSERT
240 __ cmpdi(CR0, pre_val, 0);
241 __ asm_assert_ne("null oop is not allowed");
242 #endif // ASSERT
243 } else {
244 __ cmpdi(CR0, pre_val, 0);
245 __ beq(CR0, skip_barrier);
246 }
247 } else {
248 // Load from the reference address to determine the reference's current value (before the store is being performed).
249 // Contrary to the given value in "preloaded mode", it is not necessary to preserve it.
250 assert(decorators == 0, "decorator set must be empty");
251 assert(base != noreg, "base must be a register");
252 assert(!ind_or_offs.is_register() || ind_or_offs.as_register() != noreg, "ind_or_offs must be a register");
253 if (UseCompressedOops) {
254 __ lwz(pre_val, ind_or_offs, base);
255 } else {
256 __ ld(pre_val, ind_or_offs, base);
257 }
258
259 __ cmpdi(CR0, pre_val, 0);
260 __ beq(CR0, skip_barrier);
261
262 if (UseCompressedOops) {
263 __ decode_heap_oop_not_null(pre_val);
264 }
265 }
266
267 /* ==== Try to enqueue the to-be-stored value directly into thread's local SATB mark queue ==== */
268 {
269 Label runtime;
270 Register Rbuffer = tmp1, Rindex = tmp2;
271
272 // Check whether the queue has enough capacity to store another oop.
273 // If not, jump to the runtime to commit the buffer and to allocate a new one.
274 // (The buffer's index corresponds to the amount of remaining free space.)
275 __ ld(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
276 __ cmpdi(CR0, Rindex, 0);
277 __ beq(CR0, runtime); // If index == 0 (buffer is full), goto runtime.
278
279 // Capacity suffices. Decrement the queue's size by the size of one oop.
280 // (The buffer is filled contrary to the heap's growing direction, i.e., it is filled downwards.)
281 __ addi(Rindex, Rindex, -wordSize);
282 __ std(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
283
284 // Enqueue the previous value and skip the invocation of the runtime.
285 __ ld(Rbuffer, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
286 __ stdx(pre_val, Rbuffer, Rindex);
287 __ b(skip_barrier);
288
289 __ bind(runtime);
290 }
291
292 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
293 // Save to-be-preserved registers.
294 int nbytes_save = 0;
295
296 if (needs_frame) {
297 if (preserve_gp_registers) {
298 nbytes_save = (preserve_fp_registers
299 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
300 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
301 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
302 }
303
304 __ save_LR(tmp1);
305 __ push_frame_reg_args(nbytes_save, tmp2);
306 }
307
308 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
309 assert(pre_val_save != noreg, "nv_save must not be noreg");
310
311 // 'pre_val' register must be saved manually unless general-purpose are preserved in general.
312 __ mr(pre_val_save, pre_val);
313 }
314
315 // Invoke runtime.
316 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
317
318 // Restore to-be-preserved registers.
319 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
320 __ mr(pre_val, pre_val_save);
321 }
322
323 if (needs_frame) {
324 __ pop_frame();
325 __ restore_LR(tmp1);
326
327 if (preserve_gp_registers) {
328 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
329 }
330 }
331
332 __ bind(skip_barrier);
333 }
334
335 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler *masm, Register dst, Register tmp) {
336 __ block_comment("resolve_forward_pointer_not_null (shenandoahgc) {");
337
338 Register tmp1 = tmp,
339 R0_tmp2 = R0;
340 assert_different_registers(dst, tmp1, R0_tmp2, noreg);
341
342 // If the object has been evacuated, the mark word layout is as follows:
343 // | forwarding pointer (62-bit) | '11' (2-bit) |
344
345 // The invariant that stack/thread pointers have the lowest two bits cleared permits retrieving
346 // the forwarding pointer solely by inversing the lowest two bits.
347 // This invariant follows inevitably from hotspot's minimal alignment.
348 assert(markWord::marked_value <= (unsigned long) MinObjAlignmentInBytes,
349 "marked value must not be higher than hotspot's minimal alignment");
350
351 Label done;
352
353 // Load the object's mark word.
354 __ ld(tmp1, oopDesc::mark_offset_in_bytes(), dst);
355
356 // Load the bit mask for the lock bits.
357 __ li(R0_tmp2, markWord::lock_mask_in_place);
358
359 // Check whether all bits matching the bit mask are set.
360 // If that is the case, the object has been evacuated and the most significant bits form the forward pointer.
361 __ andc_(R0_tmp2, R0_tmp2, tmp1);
362
363 assert(markWord::lock_mask_in_place == markWord::marked_value,
364 "marked value must equal the value obtained when all lock bits are being set");
365 __ xori(tmp1, tmp1, markWord::lock_mask_in_place);
366 __ isel(dst, CR0, Assembler::equal, false, tmp1);
367
368 __ bind(done);
369 __ block_comment("} resolve_forward_pointer_not_null (shenandoahgc)");
370 }
371
372 // base: Base register of the reference's address.
373 // ind_or_offs: Index or offset of the reference's address (load mode).
374 // dst: Reference's address. In case the object has been evacuated, this is the to-space version
375 // of that object.
376 void ShenandoahBarrierSetAssembler::load_reference_barrier_impl(
377 MacroAssembler *masm, DecoratorSet decorators,
378 Register base, RegisterOrConstant ind_or_offs,
379 Register dst,
380 Register tmp1, Register tmp2,
381 MacroAssembler::PreservationLevel preservation_level) {
382 if (ind_or_offs.is_register()) {
383 assert_different_registers(tmp1, tmp2, base, ind_or_offs.as_register(), dst, noreg);
384 } else {
385 assert_different_registers(tmp1, tmp2, base, dst, noreg);
386 }
387
388 Label skip_barrier;
389
390 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
391 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
392 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
393 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
394 bool is_narrow = UseCompressedOops && !is_native;
395
396 /* ==== Check whether heap is stable ==== */
397 __ lbz(tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
398
399 if (is_strong) {
400 // For strong references, the heap is considered stable if "has forwarded" is not active.
401 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
402 __ beq(CR0, skip_barrier);
403 #ifdef ASSERT
404 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
405 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
406 __ asm_assert_ne("'has forwarded' is missing");
407 #endif // ASSERT
408 } else {
409 // For all non-strong references, the heap is considered stable if not any of "has forwarded",
410 // "root set processing", and "weak reference processing" is active.
411 // The additional phase conditions are in place to avoid the resurrection of weak references (see JDK-8266440).
412 Label skip_fastpath;
413 __ andi_(tmp1, tmp2, ShenandoahHeap::WEAK_ROOTS);
414 __ bne(CR0, skip_fastpath);
415
416 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
417 __ beq(CR0, skip_barrier);
418 #ifdef ASSERT
419 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
420 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
421 __ asm_assert_ne("'has forwarded' is missing");
422 #endif // ASSERT
423
424 __ bind(skip_fastpath);
425 }
426
427 /* ==== Check whether region is in collection set ==== */
428 if (is_strong) {
429 // Shenandoah stores metadata on regions in a continuous area of memory in which a single byte corresponds to
430 // an entire region of the shenandoah heap. At present, only the least significant bit is of significance
431 // and indicates whether the region is part of the collection set.
432 //
433 // All regions are of the same size and are always aligned by a power of two.
434 // Any address can thus be shifted by a fixed number of bits to retrieve the address prefix shared by
435 // all objects within that region (region identification bits).
436 //
437 // | unused bits | region identification bits | object identification bits |
438 // (Region size depends on a couple of criteria, such as page size, user-provided arguments and the max heap size.
439 // The number of object identification bits can thus not be determined at compile time.)
440 //
441 // ------------------------------------------------------- <--- cs (collection set) base address
442 // | lost space due to heap space base address -> 'ShenandoahHeap::in_cset_fast_test_addr()'
443 // | (region identification bits contain heap base offset)
444 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
445 // | collection set in the proper -> shift: 'region_size_bytes_shift_jint()'
446 // |
447 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
448 // + number of regions
449 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
450 __ srdi(tmp1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
451 __ lbzx(tmp2, tmp1, tmp2);
452 __ andi_(tmp2, tmp2, 1);
453 __ beq(CR0, skip_barrier);
454 }
455
456 /* ==== Invoke runtime ==== */
457 // Save to-be-preserved registers.
458 int nbytes_save = 0;
459
460 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
461 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
462 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
463
464 if (needs_frame) {
465 if (preserve_gp_registers) {
466 nbytes_save = (preserve_fp_registers
467 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
468 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
469 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
470 }
471
472 __ save_LR(tmp1);
473 __ push_frame_reg_args(nbytes_save, tmp1);
474 }
475
476 // Calculate the reference's absolute address.
477 __ add(R4_ARG2, ind_or_offs, base);
478
479 // Invoke runtime.
480 address jrt_address = nullptr;
481
482 if (is_strong) {
483 if (is_narrow) {
484 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
485 } else {
486 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
487 }
488 } else if (is_weak) {
489 if (is_narrow) {
490 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
491 } else {
492 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
493 }
494 } else {
495 assert(is_phantom, "only remaining strength");
496 assert(!is_narrow, "phantom access cannot be narrow");
497 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
498 }
499 assert(jrt_address != nullptr, "jrt routine cannot be found");
500
501 __ call_VM_leaf(jrt_address, dst /* reference */, R4_ARG2 /* reference address */);
502
503 // Restore to-be-preserved registers.
504 if (preserve_gp_registers) {
505 __ mr(R0, R3_RET);
506 } else {
507 __ mr_if_needed(dst, R3_RET);
508 }
509
510 if (needs_frame) {
511 __ pop_frame();
512 __ restore_LR(tmp1);
513
514 if (preserve_gp_registers) {
515 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
516 __ mr(dst, R0);
517 }
518 }
519
520 __ bind(skip_barrier);
521 }
522
523 // base: Base register of the reference's address.
524 // ind_or_offs: Index or offset of the reference's address.
525 // L_handle_null: An optional label that will be jumped to if the reference is null.
526 void ShenandoahBarrierSetAssembler::load_at(
527 MacroAssembler *masm, DecoratorSet decorators, BasicType type,
528 Register base, RegisterOrConstant ind_or_offs, Register dst,
529 Register tmp1, Register tmp2,
530 MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
531 // Register must not clash, except 'base' and 'dst'.
532 if (ind_or_offs.is_register()) {
533 if (base != noreg) {
534 assert_different_registers(tmp1, tmp2, base, ind_or_offs.register_or_noreg(), R0, noreg);
535 }
536 assert_different_registers(tmp1, tmp2, dst, ind_or_offs.register_or_noreg(), R0, noreg);
537 } else {
538 if (base == noreg) {
539 assert_different_registers(tmp1, tmp2, base, R0, noreg);
540 }
541 assert_different_registers(tmp1, tmp2, dst, R0, noreg);
542 }
543
544 /* ==== Apply load barrier, if required ==== */
545 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
546 assert(is_reference_type(type), "need_load_reference_barrier must check whether type is a reference type");
547
548 // If 'dst' clashes with either 'base' or 'ind_or_offs', use an intermediate result register
549 // to keep the values of those alive until the load reference barrier is applied.
550 Register intermediate_dst = (dst == base || (ind_or_offs.is_register() && dst == ind_or_offs.as_register()))
551 ? tmp2
552 : dst;
553
554 BarrierSetAssembler::load_at(masm, decorators, type,
555 base, ind_or_offs,
556 intermediate_dst,
557 tmp1, noreg,
558 preservation_level, L_handle_null);
559
560 load_reference_barrier(masm, decorators,
561 base, ind_or_offs,
562 intermediate_dst,
563 tmp1, R0,
564 preservation_level);
565
566 __ mr_if_needed(dst, intermediate_dst);
567 } else {
568 BarrierSetAssembler::load_at(masm, decorators, type,
569 base, ind_or_offs,
570 dst,
571 tmp1, tmp2,
572 preservation_level, L_handle_null);
573 }
574
575 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
576 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
577 if (ShenandoahSATBBarrier) {
578 __ block_comment("keep_alive_barrier (shenandoahgc) {");
579 satb_barrier_impl(masm, 0, noreg, noreg, dst, tmp1, tmp2, preservation_level);
580 __ block_comment("} keep_alive_barrier (shenandoahgc)");
581 }
582 }
583 }
584
585 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
586 assert(ShenandoahCardBarrier, "Should have been checked by caller");
587 assert_different_registers(base, tmp, R0);
588
589 if (ind_or_offs.is_constant()) {
590 __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
591 } else {
592 __ add(base, ind_or_offs.as_register(), base);
593 }
594
595 __ ld(tmp, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread); /* tmp = *[R16_thread + card_table_offset] */
596 __ srdi(base, base, CardTable::card_shift());
597 __ li(R0, CardTable::dirty_card_val());
598 __ stbx(R0, tmp, base);
599 }
600
601 // base: Base register of the reference's address.
602 // ind_or_offs: Index or offset of the reference's address.
603 // val: To-be-stored value/reference's new value.
604 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
605 Register base, RegisterOrConstant ind_or_offs, Register val,
606 Register tmp1, Register tmp2, Register tmp3,
607 MacroAssembler::PreservationLevel preservation_level) {
608 // 1: non-reference types require no barriers
609 if (!is_reference_type(type)) {
610 BarrierSetAssembler::store_at(masm, decorators, type,
611 base, ind_or_offs,
612 val,
613 tmp1, tmp2, tmp3,
614 preservation_level);
615 return;
616 }
617
618 bool storing_non_null = (val != noreg);
619
620 // 2: pre-barrier: SATB needs the previous value
621 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
622 satb_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
623 }
624
625 // Store!
626 BarrierSetAssembler::store_at(masm, decorators, type,
627 base, ind_or_offs,
628 val,
629 tmp1, tmp2, tmp3,
630 preservation_level);
631
632 // 3: post-barrier: card barrier needs store address
633 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
634 card_barrier(masm, base, ind_or_offs, tmp1);
635 }
636 }
637
638 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
639 Register dst, Register jni_env, Register obj,
640 Register tmp, Label &slowpath) {
641 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
642
643 assert_different_registers(jni_env, obj, tmp);
644
645 Label done;
646
647 // Fast path: Reference is null (JNI tags are zero for null pointers).
648 __ cmpdi(CR0, obj, 0);
649 __ beq(CR0, done);
650
651 // Resolve jobject using standard implementation.
652 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
653
654 // Check whether heap is stable.
655 __ lbz(tmp,
656 in_bytes(ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()),
657 jni_env);
658
659 __ andi_(tmp, tmp, ShenandoahHeap::EVACUATION | ShenandoahHeap::HAS_FORWARDED);
660 __ bne(CR0, slowpath);
661
662 __ bind(done);
663 __ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
664 }
665
666 void ShenandoahBarrierSetAssembler::try_resolve_weak_handle(MacroAssembler *masm, Register obj,
667 Register tmp, Label &slow_path) {
668 __ block_comment("try_resolve_weak_handle (shenandoahgc) {");
669
670 assert_different_registers(obj, tmp);
671
672 Label done;
673
674 // Resolve weak handle using the standard implementation.
675 BarrierSetAssembler::try_resolve_weak_handle(masm, obj, tmp, slow_path);
676
677 // Check if the reference is null, and if it is, take the fast path.
678 __ cmpdi(CR0, obj, 0);
679 __ beq(CR0, done);
680
681 // Check if the heap is under weak-reference/roots processing, in
682 // which case we need to take the slow path.
683 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
684 __ andi_(tmp, tmp, ShenandoahHeap::WEAK_ROOTS);
685 __ bne(CR0, slow_path);
686 __ bind(done);
687
688 __ block_comment("} try_resolve_weak_handle (shenandoahgc)");
689 }
690
691 // Special shenandoah CAS implementation that handles false negatives due
692 // to concurrent evacuation. That is, the CAS operation is intended to succeed in
693 // the following scenarios (success criteria):
694 // s1) The reference pointer ('base_addr') equals the expected ('expected') pointer.
695 // s2) The reference pointer refers to the from-space version of an already-evacuated
696 // object, whereas the expected pointer refers to the to-space version of the same object.
697 // Situations in which the reference pointer refers to the to-space version of an object
698 // and the expected pointer refers to the from-space version of the same object can not occur due to
699 // shenandoah's strong to-space invariant. This also implies that the reference stored in 'new_val'
700 // can not refer to the from-space version of an already-evacuated object.
701 //
702 // To guarantee correct behavior in concurrent environments, two races must be addressed:
703 // r1) A concurrent thread may heal the reference pointer (i.e., it is no longer referring to the
704 // from-space version but to the to-space version of the object in question).
705 // In this case, the CAS operation should succeed.
706 // r2) A concurrent thread may mutate the reference (i.e., the reference pointer refers to an entirely different object).
707 // In this case, the CAS operation should fail.
708 //
709 // By default, the value held in the 'result' register is zero to indicate failure of CAS,
710 // non-zero to indicate success. If 'is_cae' is set, the result is the most recently fetched
711 // value from 'base_addr' rather than a boolean success indicator.
712 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler *masm, Register base_addr,
713 Register expected, Register new_val, Register tmp1, Register tmp2,
714 bool is_cae, Register result) {
715 __ block_comment("cmpxchg_oop (shenandoahgc) {");
716
717 assert_different_registers(base_addr, new_val, tmp1, tmp2, result, R0);
718 assert_different_registers(base_addr, expected, tmp1, tmp2, result, R0);
719
720 // Potential clash of 'success_flag' and 'tmp' is being accounted for.
721 Register success_flag = is_cae ? noreg : result,
722 current_value = is_cae ? result : tmp1,
723 tmp = is_cae ? tmp1 : result,
724 initial_value = tmp2;
725
726 Label done, step_four;
727
728 __ bind(step_four);
729
730 /* ==== Step 1 ("Standard" CAS) ==== */
731 // Fast path: The values stored in 'expected' and 'base_addr' are equal.
732 // Given that 'expected' must refer to the to-space object of an evacuated object (strong to-space invariant),
733 // no special processing is required.
734 if (UseCompressedOops) {
735 __ cmpxchgw(CR0, current_value, expected, new_val, base_addr, MacroAssembler::MemBarNone,
736 false, success_flag, nullptr, true);
737 } else {
738 __ cmpxchgd(CR0, current_value, expected, new_val, base_addr, MacroAssembler::MemBarNone,
739 false, success_flag, nullptr, true);
740 }
741
742 // Skip the rest of the barrier if the CAS operation succeeds immediately.
743 // If it does not, the value stored at the address is either the from-space pointer of the
744 // referenced object (success criteria s2)) or simply another object.
745 __ beq(CR0, done);
746
747 /* ==== Step 2 (Null check) ==== */
748 // The success criteria s2) cannot be matched with a null pointer
749 // (null pointers cannot be subject to concurrent evacuation). The failure of the CAS operation is thus legitimate.
750 __ cmpdi(CR0, current_value, 0);
751 __ beq(CR0, done);
752
753 /* ==== Step 3 (reference pointer refers to from-space version; success criteria s2)) ==== */
754 // To check whether the reference pointer refers to the from-space version, the forward
755 // pointer of the object referred to by the reference is resolved and compared against the expected pointer.
756 // If this check succeed, another CAS operation is issued with the from-space pointer being the expected pointer.
757 //
758 // Save the potential from-space pointer.
759 __ mr(initial_value, current_value);
760
761 // Resolve forward pointer.
762 if (UseCompressedOops) { __ decode_heap_oop_not_null(current_value); }
763 resolve_forward_pointer_not_null(masm, current_value, tmp);
764 if (UseCompressedOops) { __ encode_heap_oop_not_null(current_value); }
765
766 if (!is_cae) {
767 // 'success_flag' was overwritten by call to 'resovle_forward_pointer_not_null'.
768 // Load zero into register for the potential failure case.
769 __ li(success_flag, 0);
770 }
771 __ cmpd(CR0, current_value, expected);
772 __ bne(CR0, done);
773
774 // Discard fetched value as it might be a reference to the from-space version of an object.
775 if (UseCompressedOops) {
776 __ cmpxchgw(CR0, R0, initial_value, new_val, base_addr, MacroAssembler::MemBarNone,
777 false, success_flag);
778 } else {
779 __ cmpxchgd(CR0, R0, initial_value, new_val, base_addr, MacroAssembler::MemBarNone,
780 false, success_flag);
781 }
782
783 /* ==== Step 4 (Retry CAS with to-space pointer (success criteria s2) under race r1)) ==== */
784 // The reference pointer could have been healed whilst the previous CAS operation was being performed.
785 // Another CAS operation must thus be issued with the to-space pointer being the expected pointer.
786 // If that CAS operation fails as well, race r2) must have occurred, indicating that
787 // the operation failure is legitimate.
788 //
789 // To keep the code's size small and thus improving cache (icache) performance, this highly
790 // unlikely case should be handled by the smallest possible code. Instead of emitting a third,
791 // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
792 // (passed arguments are identical).
793 //
794 // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
795 // to fail. Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
796 // It is thus important to ensure that a re-execution of those steps does not put program correctness
797 // at risk:
798 // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
799 // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
800 // fails. Unless the reference has been updated in the meanwhile once again, this is
801 // guaranteed to be the case.
802 // In case of a concurrent update, the CAS would be retried again. This is legitimate
803 // in terms of program correctness (even though it is not desired).
804 __ bne(CR0, step_four);
805
806 __ bind(done);
807 __ block_comment("} cmpxchg_oop (shenandoahgc)");
808 }
809
810 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
811 Register addr, Register count, Register preserve) {
812 assert(ShenandoahCardBarrier, "Should have been checked by caller");
813 assert_different_registers(addr, count, R0);
814
815 Label L_skip_loop, L_store_loop;
816
817 __ sldi_(count, count, LogBytesPerHeapOop);
818
819 // Zero length? Skip.
820 __ beq(CR0, L_skip_loop);
821
822 __ addi(count, count, -BytesPerHeapOop);
823 __ add(count, addr, count);
824 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
825 __ srdi(addr, addr, CardTable::card_shift());
826 __ srdi(count, count, CardTable::card_shift());
827 __ subf(count, addr, count);
828 __ ld(R0, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
829 __ add(addr, addr, R0);
830 __ addi(count, count, 1);
831 __ li(R0, 0);
832 __ mtctr(count);
833
834 // Byte store loop
835 __ bind(L_store_loop);
836 __ stb(R0, 0, addr);
837 __ addi(addr, addr, 1);
838 __ bdnz(L_store_loop);
839 __ bind(L_skip_loop);
840 }
841
842 #undef __
843
844 #ifdef COMPILER1
845
846 #define __ ce->masm()->
847
848 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
849 __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
850
851 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
852 __ bind(*stub->entry());
853
854 // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
855 // This stub is the slowpath of that function.
856
857 assert(stub->pre_val()->is_register(), "pre_val must be a register");
858 Register pre_val = stub->pre_val()->as_register();
859
860 // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
861 // ("preloaded mode" of the store barrier).
862 if (stub->do_load()) {
863 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false);
864 }
865
866 // Fast path: Reference is null.
867 __ cmpdi(CR0, pre_val, 0);
868 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
869
870 // Argument passing via the stack.
871 __ std(pre_val, -8, R1_SP);
872
873 __ load_const_optimized(R0, bs->pre_barrier_c1_runtime_code_blob()->code_begin());
874 __ call_stub(R0);
875
876 __ b(*stub->continuation());
877 __ block_comment("} gen_pre_barrier_stub (shenandoahgc)");
878 }
879
880 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler *ce,
881 ShenandoahLoadReferenceBarrierStub *stub) {
882 __ block_comment("gen_load_reference_barrier_stub (shenandoahgc) {");
883
884 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
885 __ bind(*stub->entry());
886
887 Register obj = stub->obj()->as_register();
888 Register res = stub->result()->as_register();
889 Register addr = stub->addr()->as_pointer_register();
890 Register tmp1 = stub->tmp1()->as_register();
891 Register tmp2 = stub->tmp2()->as_register();
892 assert_different_registers(addr, res, tmp1, tmp2);
893
894 #ifdef ASSERT
895 // Ensure that 'res' is 'R3_ARG1' and contains the same value as 'obj' to reduce the number of required
896 // copy instructions.
897 assert(R3_RET == res, "res must be r3");
898 __ cmpd(CR0, res, obj);
899 __ asm_assert_eq("result register must contain the reference stored in obj");
900 #endif
901
902 DecoratorSet decorators = stub->decorators();
903
904 /* ==== Check whether region is in collection set ==== */
905 // GC status (unstable) has already been verified by 'ShenandoahBarrierSetC1::load_reference_barrier_impl'.
906 // This stub is the slowpath of that function.
907
908 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
909 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
910 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
911 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
912
913 if (is_strong) {
914 // Check whether object is in collection set.
915 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
916 __ srdi(tmp1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
917 __ lbzx(tmp2, tmp1, tmp2);
918
919 __ andi_(tmp2, tmp2, 1);
920 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
921 }
922
923 address blob_addr = nullptr;
924
925 if (is_strong) {
926 if (is_native) {
927 blob_addr = bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin();
928 } else {
929 blob_addr = bs->load_reference_barrier_strong_rt_code_blob()->code_begin();
930 }
931 } else if (is_weak) {
932 blob_addr = bs->load_reference_barrier_weak_rt_code_blob()->code_begin();
933 } else {
934 assert(is_phantom, "only remaining strength");
935 blob_addr = bs->load_reference_barrier_phantom_rt_code_blob()->code_begin();
936 }
937
938 assert(blob_addr != nullptr, "code blob cannot be found");
939
940 // Argument passing via the stack. 'obj' is passed implicitly (as asserted above).
941 __ std(addr, -8, R1_SP);
942
943 __ load_const_optimized(tmp1, blob_addr, tmp2);
944 __ call_stub(tmp1);
945
946 // 'res' is 'R3_RET'. The result is thus already in the correct register.
947
948 __ b(*stub->continuation());
949 __ block_comment("} gen_load_reference_barrier_stub (shenandoahgc)");
950 }
951
952 #undef __
953
954 #define __ sasm->
955
956 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler *sasm) {
957 __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
958
959 Label runtime, skip_barrier;
960 BarrierSet *bs = BarrierSet::barrier_set();
961
962 // Argument passing via the stack.
963 const int caller_stack_slots = 3;
964
965 Register R0_pre_val = R0;
966 __ ld(R0, -8, R1_SP);
967 Register R11_tmp1 = R11_scratch1;
968 __ std(R11_tmp1, -16, R1_SP);
969 Register R12_tmp2 = R12_scratch2;
970 __ std(R12_tmp2, -24, R1_SP);
971
972 /* ==== Check whether marking is active ==== */
973 // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
974 // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
975 __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
976
977 __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::MARKING);
978 __ beq(CR0, skip_barrier);
979
980 /* ==== Add previous value directly to thread-local SATB mark queue ==== */
981 // Check queue's capacity. Jump to runtime if no free slot is available.
982 __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
983 __ cmpdi(CR0, R12_tmp2, 0);
984 __ beq(CR0, runtime);
985
986 // Capacity suffices. Decrement the queue's size by one slot (size of one oop).
987 __ addi(R12_tmp2, R12_tmp2, -wordSize);
988 __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
989
990 // Enqueue the previous value and skip the runtime invocation.
991 __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
992 __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
993 __ b(skip_barrier);
994
995 __ bind(runtime);
996
997 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
998 // Save to-be-preserved registers.
999 const int nbytes_save = (MacroAssembler::num_volatile_regs + caller_stack_slots) * BytesPerWord;
1000 __ save_volatile_gprs(R1_SP, -nbytes_save);
1001 __ save_LR(R11_tmp1);
1002 __ push_frame_reg_args(nbytes_save, R11_tmp1);
1003
1004 // Invoke runtime.
1005 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), R0_pre_val);
1006
1007 // Restore to-be-preserved registers.
1008 __ pop_frame();
1009 __ restore_LR(R11_tmp1);
1010 __ restore_volatile_gprs(R1_SP, -nbytes_save);
1011
1012 __ bind(skip_barrier);
1013
1014 // Restore spilled registers.
1015 __ ld(R11_tmp1, -16, R1_SP);
1016 __ ld(R12_tmp2, -24, R1_SP);
1017
1018 __ blr();
1019 __ block_comment("} generate_c1_pre_barrier_runtime_stub (shenandoahgc)");
1020 }
1021
1022 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler *sasm,
1023 DecoratorSet decorators) {
1024 __ block_comment("generate_c1_load_reference_barrier_runtime_stub (shenandoahgc) {");
1025
1026 // Argument passing via the stack.
1027 const int caller_stack_slots = 1;
1028
1029 // Save to-be-preserved registers.
1030 const int nbytes_save = (MacroAssembler::num_volatile_regs - 1 // 'R3_ARG1' is skipped
1031 + caller_stack_slots) * BytesPerWord;
1032 __ save_volatile_gprs(R1_SP, -nbytes_save, true, false);
1033
1034 // Load arguments from stack.
1035 // No load required, as assured by assertions in 'ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub'.
1036 Register R3_obj = R3_ARG1;
1037 Register R4_load_addr = R4_ARG2;
1038 __ ld(R4_load_addr, -8, R1_SP);
1039
1040 Register R11_tmp = R11_scratch1;
1041
1042 /* ==== Invoke runtime ==== */
1043 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
1044 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
1045 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
1046 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
1047
1048 address jrt_address = nullptr;
1049
1050 if (is_strong) {
1051 if (is_native) {
1052 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1053 } else {
1054 if (UseCompressedOops) {
1055 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
1056 } else {
1057 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1058 }
1059 }
1060 } else if (is_weak) {
1061 assert(!is_native, "weak load reference barrier must not be called off-heap");
1062 if (UseCompressedOops) {
1063 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
1064 } else {
1065 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
1066 }
1067 } else {
1068 assert(is_phantom, "reference type must be phantom");
1069 assert(is_native, "phantom load reference barrier must be called off-heap");
1070 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
1071 }
1072 assert(jrt_address != nullptr, "load reference barrier runtime routine cannot be found");
1073
1074 __ save_LR(R11_tmp);
1075 __ push_frame_reg_args(nbytes_save, R11_tmp);
1076
1077 // Invoke runtime. Arguments are already stored in the corresponding registers.
1078 __ call_VM_leaf(jrt_address, R3_obj, R4_load_addr);
1079
1080 // Restore to-be-preserved registers.
1081 __ pop_frame();
1082 __ restore_LR(R11_tmp);
1083 __ restore_volatile_gprs(R1_SP, -nbytes_save, true, false); // Skip 'R3_RET' register.
1084
1085 __ blr();
1086 __ block_comment("} generate_c1_load_reference_barrier_runtime_stub (shenandoahgc)");
1087 }
1088
1089 #undef __
1090
1091 #endif // COMPILER1