1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2025, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "nativeInst_ppc.hpp"
42 #include "macroAssembler_ppc.hpp"
43 #include "runtime/javaThread.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "utilities/globalDefinitions.hpp"
46 #include "vm_version_ppc.hpp"
47 #ifdef COMPILER1
48 #include "c1/c1_LIRAssembler.hpp"
49 #include "c1/c1_MacroAssembler.hpp"
50 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
51 #endif
52 #ifdef COMPILER2
53 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
54 #endif
55
56 #define __ masm->
57
58 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
59 Register base, RegisterOrConstant ind_or_offs,
60 Register tmp1, Register tmp2, Register tmp3,
61 MacroAssembler::PreservationLevel preservation_level) {
62 if (ShenandoahSATBBarrier) {
63 __ block_comment("satb_barrier (shenandoahgc) {");
64 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
65 __ block_comment("} satb_barrier (shenandoahgc)");
66 }
67 }
68
69 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
70 Register base, RegisterOrConstant ind_or_offs,
71 Register dst,
72 Register tmp1, Register tmp2,
73 MacroAssembler::PreservationLevel preservation_level) {
74 if (ShenandoahLoadRefBarrier) {
75 __ block_comment("load_reference_barrier (shenandoahgc) {");
76 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
77 __ block_comment("} load_reference_barrier (shenandoahgc)");
78 }
79 }
80
81 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
82 Register src, Register dst, Register count,
83 Register preserve1, Register preserve2) {
84 Register R11_tmp = R11_scratch1;
85
86 assert_different_registers(src, dst, count, R11_tmp, noreg);
87 if (preserve1 != noreg) {
88 // Technically not required, but likely to indicate an error.
89 assert_different_registers(preserve1, preserve2);
90 }
91
92 /* ==== Check whether barrier is required (optimizations) ==== */
93 // Fast path: Component type of array is not a reference type.
94 if (!is_reference_type(type)) {
95 return;
96 }
97
98 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
99
100 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
101 // any useful information.
102 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahLoadRefBarrier) {
103 return;
104 }
105
106 __ block_comment("arraycopy_prologue (shenandoahgc) {");
107 Label skip_prologue;
108
109 // Fast path: Array is of length zero.
110 __ cmpdi(CR0, count, 0);
111 __ beq(CR0, skip_prologue);
112
113 /* ==== Check whether barrier is required (gc state) ==== */
114 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
115 R16_thread);
116
117 // The set of garbage collection states requiring barriers depends on the available barrier types and the
118 // type of the reference in question.
119 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
120 // for the garbage collector.
121 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
122 ? ShenandoahHeap::HAS_FORWARDED
123 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
124
125 __ andi_(R11_tmp, R11_tmp, required_states);
126 __ beq(CR0, skip_prologue);
127
128 /* ==== Invoke runtime ==== */
129 // Save to-be-preserved registers.
130 int highest_preserve_register_index = 0;
131 {
132 if (preserve1 != noreg && preserve1->is_volatile()) {
133 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
134 }
135 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
136 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
137 }
138
139 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
140 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
141 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
142
143 __ save_LR(R11_tmp);
144 __ push_frame_reg_args(-BytesPerWord * highest_preserve_register_index,
145 R11_tmp);
146 }
147
148 // Invoke runtime.
149 address jrt_address = nullptr;
150 if (UseCompressedOops) {
151 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
152 } else {
153 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
154 }
155 assert(jrt_address != nullptr, "jrt routine cannot be found");
156
157 __ call_VM_leaf(jrt_address, src, dst, count);
158
159 // Restore to-be-preserved registers.
160 {
161 __ pop_frame();
162 __ restore_LR(R11_tmp);
163
164 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
165 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
166 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
167
168 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
169 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
170 }
171 if (preserve1 != noreg && preserve1->is_volatile()) {
172 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
173 }
174 }
175
176 __ bind(skip_prologue);
177 __ block_comment("} arraycopy_prologue (shenandoahgc)");
178 }
179
180 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
181 Register dst, Register count,
182 Register preserve) {
183 if (ShenandoahCardBarrier && is_reference_type(type)) {
184 __ block_comment("arraycopy_epilogue (shenandoahgc) {");
185 gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
186 __ block_comment("} arraycopy_epilogue (shenandoahgc)");
187 }
188 }
189
190 // The to-be-enqueued value can either be determined
191 // - dynamically by passing the reference's address information (load mode) or
192 // - statically by passing a register the value is stored in (preloaded mode)
193 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
194 // - for incremental-update barriers.
195 //
196 // decorators: The previous value's decorator set.
197 // In "load mode", the value must equal '0'.
198 // base: Base register of the reference's address (load mode).
199 // In "preloaded mode", the register must equal 'noreg'.
200 // ind_or_offs: Index or offset of the reference's address (load mode).
201 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
202 // pre_val: Register holding the to-be-stored value (preloaded mode).
203 // In "load mode", this register acts as a temporary register and must
204 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
205 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
206 void ShenandoahBarrierSetAssembler::satb_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
207 Register base, RegisterOrConstant ind_or_offs,
208 Register pre_val,
209 Register tmp1, Register tmp2,
210 MacroAssembler::PreservationLevel preservation_level) {
211 assert(ShenandoahSATBBarrier, "Should be checked by caller");
212 assert_different_registers(tmp1, tmp2, pre_val, noreg);
213
214 Label skip_barrier;
215
216 /* ==== Determine necessary runtime invocation preservation measures ==== */
217 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
218 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
219 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
220
221 // Check whether marking is active.
222 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
223
224 __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
225 __ beq(CR0, skip_barrier);
226
227 /* ==== Determine the reference's previous value ==== */
228 bool preloaded_mode = base == noreg;
229 Register pre_val_save = noreg;
230
231 if (preloaded_mode) {
232 // Previous value has been passed to the method, so it must not be determined manually.
233 // In case 'pre_val' is a volatile register, it must be saved across the C-call
234 // as callers may depend on its value.
235 // Unless the general purposes registers are saved anyway, one of the temporary registers
236 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
237 if (!preserve_gp_registers && pre_val->is_volatile()) {
238 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
239 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
240 }
241
242 if ((decorators & IS_NOT_NULL) != 0) {
243 #ifdef ASSERT
244 __ cmpdi(CR0, pre_val, 0);
245 __ asm_assert_ne("null oop is not allowed");
246 #endif // ASSERT
247 } else {
248 __ cmpdi(CR0, pre_val, 0);
249 __ beq(CR0, skip_barrier);
250 }
251 } else {
252 // Load from the reference address to determine the reference's current value (before the store is being performed).
253 // Contrary to the given value in "preloaded mode", it is not necessary to preserve it.
254 assert(decorators == 0, "decorator set must be empty");
255 assert(base != noreg, "base must be a register");
256 assert(!ind_or_offs.is_register() || ind_or_offs.as_register() != noreg, "ind_or_offs must be a register");
257 if (UseCompressedOops) {
258 __ lwz(pre_val, ind_or_offs, base);
259 } else {
260 __ ld(pre_val, ind_or_offs, base);
261 }
262
263 __ cmpdi(CR0, pre_val, 0);
264 __ beq(CR0, skip_barrier);
265
266 if (UseCompressedOops) {
267 __ decode_heap_oop_not_null(pre_val);
268 }
269 }
270
271 /* ==== Try to enqueue the to-be-stored value directly into thread's local SATB mark queue ==== */
272 {
273 Label runtime;
274 Register Rbuffer = tmp1, Rindex = tmp2;
275
276 // Check whether the queue has enough capacity to store another oop.
277 // If not, jump to the runtime to commit the buffer and to allocate a new one.
278 // (The buffer's index corresponds to the amount of remaining free space.)
279 __ ld(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
280 __ cmpdi(CR0, Rindex, 0);
281 __ beq(CR0, runtime); // If index == 0 (buffer is full), goto runtime.
282
283 // Capacity suffices. Decrement the queue's size by the size of one oop.
284 // (The buffer is filled contrary to the heap's growing direction, i.e., it is filled downwards.)
285 __ addi(Rindex, Rindex, -wordSize);
286 __ std(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
287
288 // Enqueue the previous value and skip the invocation of the runtime.
289 __ ld(Rbuffer, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
290 __ stdx(pre_val, Rbuffer, Rindex);
291 __ b(skip_barrier);
292
293 __ bind(runtime);
294 }
295
296 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
297 // Save to-be-preserved registers.
298 int nbytes_save = 0;
299
300 if (needs_frame) {
301 if (preserve_gp_registers) {
302 nbytes_save = (preserve_fp_registers
303 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
304 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
305 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
306 }
307
308 __ save_LR(tmp1);
309 __ push_frame_reg_args(nbytes_save, tmp2);
310 }
311
312 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
313 assert(pre_val_save != noreg, "nv_save must not be noreg");
314
315 // 'pre_val' register must be saved manually unless general-purpose are preserved in general.
316 __ mr(pre_val_save, pre_val);
317 }
318
319 // Invoke runtime.
320 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
321
322 // Restore to-be-preserved registers.
323 if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
324 __ mr(pre_val, pre_val_save);
325 }
326
327 if (needs_frame) {
328 __ pop_frame();
329 __ restore_LR(tmp1);
330
331 if (preserve_gp_registers) {
332 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
333 }
334 }
335
336 __ bind(skip_barrier);
337 }
338
339 // base: Base register of the reference's address.
340 // ind_or_offs: Index or offset of the reference's address (load mode).
341 // dst: Reference's address. In case the object has been evacuated, this is the to-space version
342 // of that object.
343 void ShenandoahBarrierSetAssembler::load_reference_barrier_impl(
344 MacroAssembler *masm, DecoratorSet decorators,
345 Register base, RegisterOrConstant ind_or_offs,
346 Register dst,
347 Register tmp1, Register tmp2,
348 MacroAssembler::PreservationLevel preservation_level) {
349 if (ind_or_offs.is_register()) {
350 assert_different_registers(tmp1, tmp2, base, ind_or_offs.as_register(), dst, noreg);
351 } else {
352 assert_different_registers(tmp1, tmp2, base, dst, noreg);
353 }
354
355 Label skip_barrier;
356
357 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
358 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
359 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
360 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
361 bool is_narrow = UseCompressedOops && !is_native;
362
363 /* ==== Check whether heap is stable ==== */
364 __ lbz(tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
365
366 if (is_strong) {
367 // For strong references, the heap is considered stable if "has forwarded" is not active.
368 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
369 __ beq(CR0, skip_barrier);
370 #ifdef ASSERT
371 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
372 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
373 __ asm_assert_ne("'has forwarded' is missing");
374 #endif // ASSERT
375 } else {
376 // For all non-strong references, the heap is considered stable if not any of "has forwarded",
377 // "root set processing", and "weak reference processing" is active.
378 // The additional phase conditions are in place to avoid the resurrection of weak references (see JDK-8266440).
379 Label skip_fastpath;
380 __ andi_(tmp1, tmp2, ShenandoahHeap::WEAK_ROOTS);
381 __ bne(CR0, skip_fastpath);
382
383 __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
384 __ beq(CR0, skip_barrier);
385 #ifdef ASSERT
386 // "evacuation" -> (implies) "has forwarded". If we reach this code, "has forwarded" must thus be set.
387 __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
388 __ asm_assert_ne("'has forwarded' is missing");
389 #endif // ASSERT
390
391 __ bind(skip_fastpath);
392 }
393
394 /* ==== Check whether region is in collection set ==== */
395 if (is_strong) {
396 // Shenandoah stores metadata on regions in a continuous area of memory in which a single byte corresponds to
397 // an entire region of the shenandoah heap. At present, only the least significant bit is of significance
398 // and indicates whether the region is part of the collection set.
399 //
400 // All regions are of the same size and are always aligned by a power of two.
401 // Any address can thus be shifted by a fixed number of bits to retrieve the address prefix shared by
402 // all objects within that region (region identification bits).
403 //
404 // | unused bits | region identification bits | object identification bits |
405 // (Region size depends on a couple of criteria, such as page size, user-provided arguments and the max heap size.
406 // The number of object identification bits can thus not be determined at compile time.)
407 //
408 // ------------------------------------------------------- <--- cs (collection set) base address
409 // | lost space due to heap space base address -> 'ShenandoahHeap::in_cset_fast_test_addr()'
410 // | (region identification bits contain heap base offset)
411 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
412 // | collection set in the proper -> shift: 'region_size_bytes_shift_jint()'
413 // |
414 // |------------------------------------------------------ <--- cs base address + (heap_base >> region size shift)
415 // + number of regions
416 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
417 __ srdi(tmp1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
418 __ lbzx(tmp2, tmp1, tmp2);
419 __ andi_(tmp2, tmp2, 1);
420 __ beq(CR0, skip_barrier);
421 }
422
423 /* ==== Invoke runtime ==== */
424 // Save to-be-preserved registers.
425 int nbytes_save = 0;
426
427 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
428 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
429 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
430
431 if (needs_frame) {
432 if (preserve_gp_registers) {
433 nbytes_save = (preserve_fp_registers
434 ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
435 : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
436 __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
437 }
438
439 __ save_LR(tmp1);
440 __ push_frame_reg_args(nbytes_save, tmp1);
441 }
442
443 // Calculate the reference's absolute address.
444 __ add(R4_ARG2, ind_or_offs, base);
445
446 // Invoke runtime.
447 address jrt_address = nullptr;
448
449 if (is_strong) {
450 if (is_narrow) {
451 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
452 } else {
453 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
454 }
455 } else if (is_weak) {
456 if (is_narrow) {
457 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
458 } else {
459 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
460 }
461 } else {
462 assert(is_phantom, "only remaining strength");
463 assert(!is_narrow, "phantom access cannot be narrow");
464 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
465 }
466 assert(jrt_address != nullptr, "jrt routine cannot be found");
467
468 __ call_VM_leaf(jrt_address, dst /* reference */, R4_ARG2 /* reference address */);
469
470 // Restore to-be-preserved registers.
471 if (preserve_gp_registers) {
472 __ mr(R0, R3_RET);
473 } else {
474 __ mr_if_needed(dst, R3_RET);
475 }
476
477 if (needs_frame) {
478 __ pop_frame();
479 __ restore_LR(tmp1);
480
481 if (preserve_gp_registers) {
482 __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
483 __ mr(dst, R0);
484 }
485 }
486
487 __ bind(skip_barrier);
488 }
489
490 // base: Base register of the reference's address.
491 // ind_or_offs: Index or offset of the reference's address.
492 // L_handle_null: An optional label that will be jumped to if the reference is null.
493 void ShenandoahBarrierSetAssembler::load_at(
494 MacroAssembler *masm, DecoratorSet decorators, BasicType type,
495 Register base, RegisterOrConstant ind_or_offs, Register dst,
496 Register tmp1, Register tmp2,
497 MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
498 // Register must not clash, except 'base' and 'dst'.
499 if (ind_or_offs.is_register()) {
500 if (base != noreg) {
501 assert_different_registers(tmp1, tmp2, base, ind_or_offs.register_or_noreg(), R0, noreg);
502 }
503 assert_different_registers(tmp1, tmp2, dst, ind_or_offs.register_or_noreg(), R0, noreg);
504 } else {
505 if (base == noreg) {
506 assert_different_registers(tmp1, tmp2, base, R0, noreg);
507 }
508 assert_different_registers(tmp1, tmp2, dst, R0, noreg);
509 }
510
511 /* ==== Apply load barrier, if required ==== */
512 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
513 assert(is_reference_type(type), "need_load_reference_barrier must check whether type is a reference type");
514
515 // If 'dst' clashes with either 'base' or 'ind_or_offs', use an intermediate result register
516 // to keep the values of those alive until the load reference barrier is applied.
517 Register intermediate_dst = (dst == base || (ind_or_offs.is_register() && dst == ind_or_offs.as_register()))
518 ? tmp2
519 : dst;
520
521 BarrierSetAssembler::load_at(masm, decorators, type,
522 base, ind_or_offs,
523 intermediate_dst,
524 tmp1, noreg,
525 preservation_level, L_handle_null);
526
527 load_reference_barrier(masm, decorators,
528 base, ind_or_offs,
529 intermediate_dst,
530 tmp1, R0,
531 preservation_level);
532
533 __ mr_if_needed(dst, intermediate_dst);
534 } else {
535 BarrierSetAssembler::load_at(masm, decorators, type,
536 base, ind_or_offs,
537 dst,
538 tmp1, tmp2,
539 preservation_level, L_handle_null);
540 }
541
542 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
543 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
544 if (ShenandoahSATBBarrier) {
545 __ block_comment("keep_alive_barrier (shenandoahgc) {");
546 satb_barrier_impl(masm, 0, noreg, noreg, dst, tmp1, tmp2, preservation_level);
547 __ block_comment("} keep_alive_barrier (shenandoahgc)");
548 }
549 }
550 }
551
552 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
553 assert(ShenandoahCardBarrier, "Should have been checked by caller");
554 assert_different_registers(base, tmp, R0);
555
556 if (ind_or_offs.is_constant()) {
557 __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
558 } else {
559 __ add(base, ind_or_offs.as_register(), base);
560 }
561
562 __ ld(tmp, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread); /* tmp = *[R16_thread + card_table_offset] */
563 __ srdi(base, base, CardTable::card_shift());
564 __ li(R0, CardTable::dirty_card_val());
565 __ stbx(R0, tmp, base);
566 }
567
568 // base: Base register of the reference's address.
569 // ind_or_offs: Index or offset of the reference's address.
570 // val: To-be-stored value/reference's new value.
571 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
572 Register base, RegisterOrConstant ind_or_offs, Register val,
573 Register tmp1, Register tmp2, Register tmp3,
574 MacroAssembler::PreservationLevel preservation_level) {
575 // 1: non-reference types require no barriers
576 if (!is_reference_type(type)) {
577 BarrierSetAssembler::store_at(masm, decorators, type,
578 base, ind_or_offs,
579 val,
580 tmp1, tmp2, tmp3,
581 preservation_level);
582 return;
583 }
584
585 bool storing_non_null = (val != noreg);
586
587 // 2: pre-barrier: SATB needs the previous value
588 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
589 satb_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
590 }
591
592 // Store!
593 BarrierSetAssembler::store_at(masm, decorators, type,
594 base, ind_or_offs,
595 val,
596 tmp1, tmp2, tmp3,
597 preservation_level);
598
599 // 3: post-barrier: card barrier needs store address
600 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
601 card_barrier(masm, base, ind_or_offs, tmp1);
602 }
603 }
604
605 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
606 Register dst, Register jni_env, Register obj,
607 Register tmp, Label &slowpath) {
608 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
609
610 assert_different_registers(jni_env, obj, tmp);
611
612 Label done;
613
614 // Fast path: Reference is null (JNI tags are zero for null pointers).
615 __ cmpdi(CR0, obj, 0);
616 __ beq(CR0, done);
617
618 // Resolve jobject using standard implementation.
619 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
620
621 // Check whether heap is stable.
622 __ lbz(tmp,
623 in_bytes(ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()),
624 jni_env);
625
626 __ andi_(tmp, tmp, ShenandoahHeap::EVACUATION | ShenandoahHeap::HAS_FORWARDED);
627 __ bne(CR0, slowpath);
628
629 __ bind(done);
630 __ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
631 }
632
633 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
634 Register obj, Register tmp, Label &slow_path) {
635 __ block_comment("try_peek_weak_handle_in_nmethod (shenandoahgc) {");
636
637 assert_different_registers(weak_handle, tmp, noreg);
638 assert_different_registers(obj, tmp, noreg);
639
640
641 Label done;
642
643 // Peek weak handle using the standard implementation.
644 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
645
646 // Check if the reference is null, and if it is, take the fast path.
647 __ cmpdi(CR0, obj, 0);
648 __ beq(CR0, done);
649
650 // Check if the heap is under weak-reference/roots processing, in
651 // which case we need to take the slow path.
652 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
653 __ andi_(tmp, tmp, ShenandoahHeap::WEAK_ROOTS);
654 __ bne(CR0, slow_path);
655 __ bind(done);
656
657 __ block_comment("} try_peek_weak_handle_in_nmethod (shenandoahgc)");
658 }
659
660 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
661 Register addr, Register count, Register preserve) {
662 assert(ShenandoahCardBarrier, "Should have been checked by caller");
663 assert_different_registers(addr, count, R0);
664
665 Label L_skip_loop, L_store_loop;
666
667 __ sldi_(count, count, LogBytesPerHeapOop);
668
669 // Zero length? Skip.
670 __ beq(CR0, L_skip_loop);
671
672 __ addi(count, count, -BytesPerHeapOop);
673 __ add(count, addr, count);
674 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
675 __ srdi(addr, addr, CardTable::card_shift());
676 __ srdi(count, count, CardTable::card_shift());
677 __ subf(count, addr, count);
678 __ ld(R0, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
679 __ add(addr, addr, R0);
680 __ addi(count, count, 1);
681 __ li(R0, 0);
682 __ mtctr(count);
683
684 // Byte store loop
685 __ bind(L_store_loop);
686 __ stb(R0, 0, addr);
687 __ addi(addr, addr, 1);
688 __ bdnz(L_store_loop);
689 __ bind(L_skip_loop);
690 }
691
692 #undef __
693
694 #ifdef COMPILER1
695
696 #define __ ce->masm()->
697
698 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
699 __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
700
701 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
702 __ bind(*stub->entry());
703
704 // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
705 // This stub is the slowpath of that function.
706
707 assert(stub->pre_val()->is_register(), "pre_val must be a register");
708 Register pre_val = stub->pre_val()->as_register();
709
710 // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
711 // ("preloaded mode" of the store barrier).
712 if (stub->do_load()) {
713 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false);
714 }
715
716 // Fast path: Reference is null.
717 __ cmpdi(CR0, pre_val, 0);
718 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
719
720 // Argument passing via the stack.
721 __ std(pre_val, -8, R1_SP);
722
723 __ load_const_optimized(R0, bs->pre_barrier_c1_runtime_code_blob()->code_begin());
724 __ call_stub(R0);
725
726 __ b(*stub->continuation());
727 __ block_comment("} gen_pre_barrier_stub (shenandoahgc)");
728 }
729
730 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler *ce,
731 ShenandoahLoadReferenceBarrierStub *stub) {
732 __ block_comment("gen_load_reference_barrier_stub (shenandoahgc) {");
733
734 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
735 __ bind(*stub->entry());
736
737 Register obj = stub->obj()->as_register();
738 Register res = stub->result()->as_register();
739 Register addr = stub->addr()->as_pointer_register();
740 Register tmp1 = stub->tmp1()->as_register();
741 Register tmp2 = stub->tmp2()->as_register();
742 assert_different_registers(addr, res, tmp1, tmp2);
743
744 assert(R3_RET == res, "res must be r3");
745
746 if (res != obj) {
747 __ mr(res, obj);
748 }
749
750 DecoratorSet decorators = stub->decorators();
751
752 /* ==== Check whether region is in collection set ==== */
753 // GC status (unstable) has already been verified by 'ShenandoahBarrierSetC1::load_reference_barrier_impl'.
754 // This stub is the slowpath of that function.
755
756 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
757 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
758 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
759 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
760
761 if (is_strong) {
762 // Check whether object is in collection set.
763 __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
764 __ srdi(tmp1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
765 __ lbzx(tmp2, tmp1, tmp2);
766
767 __ andi_(tmp2, tmp2, 1);
768 __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CR0, Assembler::equal), *stub->continuation());
769 }
770
771 address blob_addr = nullptr;
772
773 if (is_strong) {
774 if (is_native) {
775 blob_addr = bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin();
776 } else {
777 blob_addr = bs->load_reference_barrier_strong_rt_code_blob()->code_begin();
778 }
779 } else if (is_weak) {
780 blob_addr = bs->load_reference_barrier_weak_rt_code_blob()->code_begin();
781 } else {
782 assert(is_phantom, "only remaining strength");
783 blob_addr = bs->load_reference_barrier_phantom_rt_code_blob()->code_begin();
784 }
785
786 assert(blob_addr != nullptr, "code blob cannot be found");
787
788 // Argument passing via the stack. 'obj' is passed implicitly (as asserted above).
789 __ std(addr, -8, R1_SP);
790
791 __ load_const_optimized(tmp1, blob_addr, tmp2);
792 __ call_stub(tmp1);
793
794 // 'res' is 'R3_RET'. The result is thus already in the correct register.
795
796 __ b(*stub->continuation());
797 __ block_comment("} gen_load_reference_barrier_stub (shenandoahgc)");
798 }
799
800 #undef __
801
802 #define __ sasm->
803
804 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler *sasm) {
805 __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
806
807 Label runtime, skip_barrier;
808 BarrierSet *bs = BarrierSet::barrier_set();
809
810 // Argument passing via the stack.
811 const int caller_stack_slots = 3;
812
813 Register R0_pre_val = R0;
814 __ ld(R0, -8, R1_SP);
815 Register R11_tmp1 = R11_scratch1;
816 __ std(R11_tmp1, -16, R1_SP);
817 Register R12_tmp2 = R12_scratch2;
818 __ std(R12_tmp2, -24, R1_SP);
819
820 /* ==== Check whether marking is active ==== */
821 // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
822 // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
823 __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
824
825 __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::MARKING);
826 __ beq(CR0, skip_barrier);
827
828 /* ==== Add previous value directly to thread-local SATB mark queue ==== */
829 // Check queue's capacity. Jump to runtime if no free slot is available.
830 __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
831 __ cmpdi(CR0, R12_tmp2, 0);
832 __ beq(CR0, runtime);
833
834 // Capacity suffices. Decrement the queue's size by one slot (size of one oop).
835 __ addi(R12_tmp2, R12_tmp2, -wordSize);
836 __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
837
838 // Enqueue the previous value and skip the runtime invocation.
839 __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
840 __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
841 __ b(skip_barrier);
842
843 __ bind(runtime);
844
845 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
846 // Save to-be-preserved registers.
847 const int nbytes_save = (MacroAssembler::num_volatile_regs + caller_stack_slots) * BytesPerWord;
848 __ save_volatile_gprs(R1_SP, -nbytes_save);
849 __ save_LR(R11_tmp1);
850 __ push_frame_reg_args(nbytes_save, R11_tmp1);
851
852 // Invoke runtime.
853 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), R0_pre_val);
854
855 // Restore to-be-preserved registers.
856 __ pop_frame();
857 __ restore_LR(R11_tmp1);
858 __ restore_volatile_gprs(R1_SP, -nbytes_save);
859
860 __ bind(skip_barrier);
861
862 // Restore spilled registers.
863 __ ld(R11_tmp1, -16, R1_SP);
864 __ ld(R12_tmp2, -24, R1_SP);
865
866 __ blr();
867 __ block_comment("} generate_c1_pre_barrier_runtime_stub (shenandoahgc)");
868 }
869
870 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler *sasm,
871 DecoratorSet decorators) {
872 __ block_comment("generate_c1_load_reference_barrier_runtime_stub (shenandoahgc) {");
873
874 // Argument passing via the stack.
875 const int caller_stack_slots = 1;
876
877 // Save to-be-preserved registers.
878 const int nbytes_save = (MacroAssembler::num_volatile_regs - 1 // 'R3_ARG1' is skipped
879 + caller_stack_slots) * BytesPerWord;
880 __ save_volatile_gprs(R1_SP, -nbytes_save, true, false);
881
882 // Load arguments from stack.
883 // No load required, as caller has already loaded obj into R3.
884 Register R3_obj = R3_ARG1;
885 Register R4_load_addr = R4_ARG2;
886 __ ld(R4_load_addr, -8, R1_SP);
887
888 Register R11_tmp = R11_scratch1;
889
890 /* ==== Invoke runtime ==== */
891 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
892 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
893 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
894 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
895
896 address jrt_address = nullptr;
897
898 if (is_strong) {
899 if (is_native) {
900 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
901 } else {
902 if (UseCompressedOops) {
903 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
904 } else {
905 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
906 }
907 }
908 } else if (is_weak) {
909 assert(!is_native, "weak load reference barrier must not be called off-heap");
910 if (UseCompressedOops) {
911 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
912 } else {
913 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
914 }
915 } else {
916 assert(is_phantom, "reference type must be phantom");
917 assert(is_native, "phantom load reference barrier must be called off-heap");
918 jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
919 }
920 assert(jrt_address != nullptr, "load reference barrier runtime routine cannot be found");
921
922 __ save_LR(R11_tmp);
923 __ push_frame_reg_args(nbytes_save, R11_tmp);
924
925 // Invoke runtime. Arguments are already stored in the corresponding registers.
926 __ call_VM_leaf(jrt_address, R3_obj, R4_load_addr);
927
928 // Restore to-be-preserved registers.
929 __ pop_frame();
930 __ restore_LR(R11_tmp);
931 __ restore_volatile_gprs(R1_SP, -nbytes_save, true, false); // Skip 'R3_RET' register.
932
933 __ blr();
934 __ block_comment("} generate_c1_load_reference_barrier_runtime_stub (shenandoahgc)");
935 }
936
937 #undef __
938
939 #endif // COMPILER1
940
941 #ifdef COMPILER2
942
943 #undef __
944 #define __ masm->
945
946 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Register addr, int disp, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
947 if (is_narrow) {
948 __ lwz(dst, disp, addr);
949 } else {
950 __ ld(dst, disp, addr);
951 }
952 if (is_acquire) {
953 __ twi_0(dst);
954 __ isync();
955 }
956
957 ShenandoahBarrierStubC2::load_post(masm, node, dst, Address(addr, disp), tmp1, tmp2, is_narrow);
958 }
959
960 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
961 Register dst, int disp, bool dst_narrow, Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
962
963 ShenandoahBarrierStubC2::store_pre(masm, node, Address(dst, disp), tmp1, tmp2, tmp3, dst_narrow);
964
965 if (dst_narrow && !src_narrow) {
966 // Need to encode into tmp, because we cannot clobber src.
967 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
968 src = __ encode_heap_oop(tmp1, src);
969 } else {
970 src = __ encode_heap_oop_not_null(tmp1, src);
971 }
972 }
973 if (dst_narrow) {
974 __ stw(src, disp, dst);
975 } else {
976 __ std(src, disp, dst);
977 }
978
979 ShenandoahBarrierStubC2::store_post(masm, node, Address(dst, disp), tmp1, tmp2);
980 }
981
982 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
983 Register oldval, Register newval, Register tmp1, Register tmp2, bool exchange, bool narrow, bool weak, bool acquire) {
984
985 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, res, tmp1, tmp2, narrow);
986
987 Register dest_current = exchange ? res : R0;
988 Label no_update;
989 int semantics = MacroAssembler::MemBarNone;
990
991 if (acquire) {
992 semantics = support_IRIW_for_not_multiple_copy_atomic_cpu ?
993 MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter;
994 }
995
996 if (!exchange) { __ li(res, 0); }
997 if (narrow) {
998 __ cmpxchgw(CR0, dest_current, oldval, newval, addr,
999 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
1000 noreg, &no_update, true, weak);
1001 } else {
1002 __ cmpxchgd(CR0, dest_current, oldval, newval, addr,
1003 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
1004 noreg, &no_update, true, weak);
1005 }
1006 if (!exchange) { __ li(res, 1); }
1007
1008 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
1009
1010 __ bind(no_update);
1011 }
1012
1013 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval, Register newval, Register addr, Register tmp1, Register tmp2) {
1014 bool is_narrow = node->bottom_type()->isa_narrowoop();
1015
1016 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, preval, tmp1, tmp2, is_narrow);
1017
1018 if (is_narrow) {
1019 __ getandsetw(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
1020 } else {
1021 __ getandsetd(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
1022 }
1023
1024 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
1025 __ isync();
1026 } else {
1027 __ sync();
1028 }
1029
1030 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
1031 }
1032
1033 #undef __
1034 #define __ masm.
1035
1036 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
1037 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1038 assert_different_registers(tmp1, tmp2, address.index(), address.base());
1039
1040 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
1041 if (address.index() == noreg) {
1042 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
1043 } else {
1044 __ add(tmp2, address.index(), address.base());
1045 if (address.disp() != 0) {
1046 __ addi(tmp2, tmp2, address.disp());
1047 }
1048 }
1049 __ srdi(tmp2, tmp2, CardTable::card_shift());
1050 __ li(R0, CardTable::dirty_card_val());
1051 __ stbx(R0, tmp2, tmp1);
1052 }
1053
1054 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
1055 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1056
1057 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(test_state));
1058 __ b(*entry());
1059
1060 // This is were the slowpath stub will return to
1061 __ bind(*continuation());
1062 }
1063
1064 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
1065 NativeInstruction* ni = nativeInstruction_at(pc);
1066 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
1067 NativeGeneralJump* jmp = nativeGeneralJump_at(pc);
1068 return jmp->jump_destination();
1069 }
1070
1071 static void check_at(bool cond, address pc, const char* msg) {
1072 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x",
1073 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3));
1074 }
1075
1076 static bool is_nop(address pc) {
1077 if (*(pc + 0) != 0x00) return false;
1078 if (*(pc + 1) != 0x00) return false;
1079 if (*(pc + 2) != 0x00) return false;
1080 if (*(pc + 3) != 0x60) return false;
1081 return true;
1082 }
1083
1084 static void insert_nop(address pc) {
1085 *reinterpret_cast<int32_t*>(pc) = 0x60000000;
1086 check_at(is_nop(pc), pc, "Should be nop");
1087 ICache::invalidate_range(pc, 4);
1088 }
1089
1090 bool ShenandoahBarrierSetAssembler::is_active(address pc) {
1091 NativeInstruction* ni = nativeInstruction_at(pc);
1092 return ni->is_jump();
1093 }
1094
1095 void ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
1096 NativeInstruction* ni = nativeInstruction_at(pc);
1097 if (ni->is_jump()) {
1098 insert_nop(pc);
1099 } else {
1100 check_at(is_nop(pc), pc, "Should already be nop");
1101 }
1102 }
1103
1104 void ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
1105 NativeInstruction* ni = nativeInstruction_at(pc);
1106 if (is_nop(pc)) {
1107 NativeGeneralJump::insert_unconditional(pc, stub_addr);
1108 } else {
1109 check_at(ni->is_jump(), pc, "Should already be jump");
1110 check_at(nativeGeneralJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
1111 }
1112 }
1113
1114 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1115 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1116 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1117
1118 __ bind(*entry());
1119
1120 // If we need to load ourselves, do it here.
1121 if (_do_load) {
1122 if (_narrow) {
1123 __ lwz(_obj, _addr.disp(), _addr.base());
1124 } else {
1125 __ ld(_obj, _addr.disp(), _addr.base());
1126 }
1127 }
1128
1129 // If the object is null, there is no point in applying barriers.
1130 maybe_far_jump_if_zero(masm, _obj);
1131
1132 // We need to make sure that loads done by callers survive across slow-path calls.
1133 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1134 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
1135 if (!_do_load || needs_both_barriers) {
1136 preserve(_obj);
1137 }
1138
1139 // Go for barriers. Barriers can return straight to continuation, as long
1140 // as another barrier is not needed and we can reach the fastpath.
1141 if (needs_both_barriers) {
1142 keepalive(masm, nullptr);
1143 lrb(masm);
1144 } else if (_needs_keep_alive_barrier) {
1145 keepalive(masm, continuation());
1146 } else if (_needs_load_ref_barrier) {
1147 lrb(masm);
1148 } else {
1149 ShouldNotReachHere();
1150 }
1151 }
1152
1153 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1154 __ cmpdi(CR0, reg, 0);
1155 // Branch to target if equal
1156 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
1157 }
1158
1159 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1160 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1161 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1162 Label L_through, L_slowpath;
1163
1164 // If another barrier is enabled as well, do a check for a specific barrier.
1165 if (_needs_load_ref_barrier) {
1166 assert(L_done == nullptr, "Should be");
1167 // Emit the unconditional branch in the first version of the method.
1168 // Let the rest of runtime figure out how to manage it.
1169 // TODO: We could have spared the over-jump if patching knew we need the inverse branch.
1170 char state_to_check = ShenandoahHeap::MARKING;
1171 Label L_over;
1172 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
1173 __ b(L_over);
1174 __ b(L_through);
1175 __ bind(L_over);
1176 }
1177
1178 // Fast-path: put object into buffer.
1179 // If buffer is already full, go slow.
1180 __ ld(_tmp1, index_offset, R16_thread);
1181 __ cmpdi(CR0, _tmp1, 0);
1182 __ beq(CR0, L_slowpath);
1183 __ addi(_tmp1, _tmp1, -wordSize);
1184 __ std(_tmp1, index_offset, R16_thread);
1185 __ ld(_tmp2, buffer_offset, R16_thread);
1186
1187 // Store the object in queue.
1188 // If object is narrow, we need to decode it before inserting.
1189 if (_narrow) {
1190 __ add(_tmp2, _tmp2, _tmp1);
1191 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1192 __ stdx(decoded, _tmp2);
1193 } else {
1194 __ stdx(_obj, _tmp2, _tmp1);
1195 }
1196
1197 // Fast-path exits here.
1198 if (L_done != nullptr) {
1199 __ b(*L_done);
1200 } else {
1201 __ b(L_through);
1202 }
1203
1204 // Slow-path: call runtime to handle.
1205 __ bind(L_slowpath);
1206
1207 {
1208 SaveLiveRegisters slr(&masm, this);
1209
1210 // Go to runtime and handle the rest there.
1211 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1212 }
1213
1214 if (L_done != nullptr) {
1215 __ b(*L_done);
1216 } else {
1217 __ bind(L_through);
1218 }
1219 }
1220
1221 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1222 Label L_slow;
1223
1224 // If weak references are being processed, weak/phantom loads need to go slow,
1225 // regardless of their cset status.
1226 if (_needs_load_ref_weak_barrier) {
1227 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
1228 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
1229 __ b(L_slow);
1230 }
1231
1232 if (_needs_keep_alive_barrier) {
1233 // Emit the unconditional branch in the first version of the method.
1234 // Let the rest of runtime figure out how to manage it.
1235 // TODO: We could have spared the over-jump if patching knew we need the inverse branch.
1236 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1237 Label L_over;
1238 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
1239 __ b(L_over);
1240 __ b(*continuation());
1241 __ bind(L_over);
1242 }
1243
1244 // Cset-check. Fall-through to slow if in collection set.
1245 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1246 if (_narrow) {
1247 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1248 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1249 } else {
1250 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1251 }
1252 __ lbzx(_tmp2, _tmp2, _tmp1);
1253 maybe_far_jump_if_zero(masm, _tmp2);
1254
1255 // Slow path
1256 __ bind(L_slow);
1257
1258 // Obj is the result, need to temporarily stop preserving it.
1259 bool is_obj_preserved = is_preserved(_obj);
1260 if (is_obj_preserved) {
1261 dont_preserve(_obj);
1262 }
1263 {
1264 SaveLiveRegisters slr(&masm, this);
1265
1266 // Shuffle in the arguments. The end result should be:
1267 // c_rarg0 <-- obj
1268 // c_rarg1 <-- lea(addr)
1269 Register c_rarg0 = R3_ARG1;
1270 Register c_rarg1 = R4_ARG2;
1271 if (c_rarg0 == _obj) {
1272 __ addi(c_rarg1, _addr.base(), _addr.disp());
1273 } else if (c_rarg1 == _obj) {
1274 __ mr(_tmp1, c_rarg1);
1275 __ addi(c_rarg1, _addr.base(), _addr.disp());
1276 __ mr(c_rarg0, _tmp1);
1277 } else {
1278 assert_different_registers(c_rarg1, _obj);
1279 __ addi(c_rarg1, _addr.base(), _addr.disp());
1280 __ mr(c_rarg0, _obj);
1281 }
1282
1283 // Go to runtime and handle the rest there.
1284 __ call_VM_leaf(lrb_runtime_entry_addr(), c_rarg0, c_rarg1);
1285
1286 // Save the result where needed.
1287 if (_obj != R3_RET) {
1288 __ mr(_obj, R3_RET);
1289 }
1290 }
1291 if (is_obj_preserved) {
1292 preserve(_obj);
1293 }
1294
1295 __ b(*continuation());
1296 }
1297
1298 int ShenandoahBarrierStubC2::available_gp_registers() {
1299 Unimplemented(); // Not used
1300 return 0;
1301 }
1302
1303 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1304 Unimplemented(); // Not used
1305 return true;
1306 }
1307
1308 void ShenandoahBarrierStubC2::post_init() {
1309 // Do nothing.
1310 }
1311
1312 #endif // COMPILER2