20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "gc/shared/gcArguments.hpp"
27 #include "gc/shared/gc_globals.hpp"
28 #include "macroAssembler_ppc.hpp"
29 #include "precompiled.hpp"
30 #include "asm/macroAssembler.inline.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
33 #include "gc/shenandoah/shenandoahForwarding.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahRuntime.hpp"
38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
39 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "utilities/globalDefinitions.hpp"
44 #include "vm_version_ppc.hpp"
45 #ifdef COMPILER1
46 #include "c1/c1_LIRAssembler.hpp"
47 #include "c1/c1_MacroAssembler.hpp"
48 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
49 #endif
50
51 #define __ masm->
52
53 void ShenandoahBarrierSetAssembler::satb_write_barrier(MacroAssembler *masm,
54 Register base, RegisterOrConstant ind_or_offs,
55 Register tmp1, Register tmp2, Register tmp3,
56 MacroAssembler::PreservationLevel preservation_level) {
57 if (ShenandoahSATBBarrier) {
58 __ block_comment("satb_write_barrier (shenandoahgc) {");
59 satb_write_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
73 satb_write_barrier_impl(masm, decorators, noreg, noreg, val, tmp1, tmp2, preservation_level);
74 __ block_comment("} iu_barrier (shenandoahgc)");
75 }
76 }
77
78 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
79 Register base, RegisterOrConstant ind_or_offs,
80 Register dst,
81 Register tmp1, Register tmp2,
82 MacroAssembler::PreservationLevel preservation_level) {
83 if (ShenandoahLoadRefBarrier) {
84 __ block_comment("load_reference_barrier (shenandoahgc) {");
85 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
86 __ block_comment("} load_reference_barrier (shenandoahgc)");
87 }
88 }
89
90 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
91 Register src, Register dst, Register count,
92 Register preserve1, Register preserve2) {
93 __ block_comment("arraycopy_prologue (shenandoahgc) {");
94
95 Register R11_tmp = R11_scratch1;
96
97 assert_different_registers(src, dst, count, R11_tmp, noreg);
98 if (preserve1 != noreg) {
99 // Technically not required, but likely to indicate an error.
100 assert_different_registers(preserve1, preserve2);
101 }
102
103 /* ==== Check whether barrier is required (optimizations) ==== */
104 // Fast path: Component type of array is not a reference type.
105 if (!is_reference_type(type)) {
106 return;
107 }
108
109 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
110
111 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
112 // any useful information.
113 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahIUBarrier && !ShenandoahLoadRefBarrier) {
114 return;
115 }
116
117 Label skip_prologue;
118
119 // Fast path: Array is of length zero.
120 __ cmpdi(CCR0, count, 0);
121 __ beq(CCR0, skip_prologue);
122
123 /* ==== Check whether barrier is required (gc state) ==== */
124 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
125 R16_thread);
126
127 // The set of garbage collection states requiring barriers depends on the available barrier types and the
128 // type of the reference in question.
129 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
130 // for the garbage collector.
131 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
132 ? ShenandoahHeap::HAS_FORWARDED
133 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
134
135 __ andi_(R11_tmp, R11_tmp, required_states);
136 __ beq(CCR0, skip_prologue);
137
138 /* ==== Invoke runtime ==== */
139 // Save to-be-preserved registers.
140 int highest_preserve_register_index = 0;
141 {
142 if (preserve1 != noreg && preserve1->is_volatile()) {
143 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
144 }
145 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
146 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
147 }
148
149 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
150 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
151 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
152
153 __ save_LR_CR(R11_tmp);
170 {
171 __ pop_frame();
172 __ restore_LR_CR(R11_tmp);
173
174 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
175 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
176 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
177
178 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
179 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
180 }
181 if (preserve1 != noreg && preserve1->is_volatile()) {
182 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
183 }
184 }
185
186 __ bind(skip_prologue);
187 __ block_comment("} arraycopy_prologue (shenandoahgc)");
188 }
189
190 // The to-be-enqueued value can either be determined
191 // - dynamically by passing the reference's address information (load mode) or
192 // - statically by passing a register the value is stored in (preloaded mode)
193 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
194 // - for incremental-update barriers.
195 //
196 // decorators: The previous value's decorator set.
197 // In "load mode", the value must equal '0'.
198 // base: Base register of the reference's address (load mode).
199 // In "preloaded mode", the register must equal 'noreg'.
200 // ind_or_offs: Index or offset of the reference's address (load mode).
201 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
202 // pre_val: Register holding the to-be-stored value (preloaded mode).
203 // In "load mode", this register acts as a temporary register and must
204 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
205 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
206 void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
207 Register base, RegisterOrConstant ind_or_offs,
208 Register pre_val,
209 Register tmp1, Register tmp2,
210 MacroAssembler::PreservationLevel preservation_level) {
211 assert_different_registers(tmp1, tmp2, pre_val, noreg);
212
213 Label skip_barrier;
214
215 /* ==== Determine necessary runtime invocation preservation measures ==== */
216 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
217 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
218 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
219
220 // Check whether marking is active.
221 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
222
223 __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
224 __ beq(CCR0, skip_barrier);
225
226 /* ==== Determine the reference's previous value ==== */
227 bool preloaded_mode = base == noreg;
228 Register pre_val_save = noreg;
229
230 if (preloaded_mode) {
231 // Previous value has been passed to the method, so it must not be determined manually.
232 // In case 'pre_val' is a volatile register, it must be saved across the C-call
233 // as callers may depend on its value.
234 // Unless the general purposes registers are saved anyway, one of the temporary registers
235 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
236 if (!preserve_gp_registers && pre_val->is_volatile()) {
237 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
238 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
239 }
240
241 if ((decorators & IS_NOT_NULL) != 0) {
242 #ifdef ASSERT
243 __ cmpdi(CCR0, pre_val, 0);
569 base, ind_or_offs,
570 intermediate_dst,
571 tmp1, R0,
572 preservation_level);
573
574 __ mr_if_needed(dst, intermediate_dst);
575 } else {
576 BarrierSetAssembler::load_at(masm, decorators, type,
577 base, ind_or_offs,
578 dst,
579 tmp1, tmp2,
580 preservation_level, L_handle_null);
581 }
582
583 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
584 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
585 iu_barrier(masm, dst, tmp1, tmp2, preservation_level);
586 }
587 }
588
589 // base: Base register of the reference's address.
590 // ind_or_offs: Index or offset of the reference's address.
591 // val: To-be-stored value/reference's new value.
592 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
593 Register base, RegisterOrConstant ind_or_offs, Register val,
594 Register tmp1, Register tmp2, Register tmp3,
595 MacroAssembler::PreservationLevel preservation_level) {
596 if (is_reference_type(type)) {
597 if (ShenandoahSATBBarrier) {
598 satb_write_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
599 }
600
601 if (ShenandoahIUBarrier && val != noreg) {
602 iu_barrier(masm, val, tmp1, tmp2, preservation_level, decorators);
603 }
604 }
605
606 BarrierSetAssembler::store_at(masm, decorators, type,
607 base, ind_or_offs,
608 val,
609 tmp1, tmp2, tmp3,
610 preservation_level);
611 }
612
613 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
614 Register dst, Register jni_env, Register obj,
615 Register tmp, Label &slowpath) {
616 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
617
618 assert_different_registers(jni_env, obj, tmp);
619
620 Label done;
621
622 // Fast path: Reference is null (JNI tags are zero for null pointers).
623 __ cmpdi(CCR0, obj, 0);
624 __ beq(CCR0, done);
625
626 // Resolve jobject using standard implementation.
627 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
628
629 // Check whether heap is stable.
630 __ lbz(tmp,
740 // unlikely case should be handled by the smallest possible code. Instead of emitting a third,
741 // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
742 // (passed arguments are identical).
743 //
744 // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
745 // to fail. Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
746 // It is thus important to ensure that a re-execution of those steps does not put program correctness
747 // at risk:
748 // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
749 // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
750 // fails. Unless the reference has been updated in the meanwhile once again, this is
751 // guaranteed to be the case.
752 // In case of a concurrent update, the CAS would be retried again. This is legitimate
753 // in terms of program correctness (even though it is not desired).
754 __ bne(CCR0, step_four);
755
756 __ bind(done);
757 __ block_comment("} cmpxchg_oop (shenandoahgc)");
758 }
759
760 #undef __
761
762 #ifdef COMPILER1
763
764 #define __ ce->masm()->
765
766 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
767 __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
768
769 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
770 __ bind(*stub->entry());
771
772 // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
773 // This stub is the slowpath of that function.
774
775 assert(stub->pre_val()->is_register(), "pre_val must be a register");
776 Register pre_val = stub->pre_val()->as_register();
777
778 // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
779 // ("preloaded mode" of the store barrier).
875 __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
876
877 Label runtime, skip_barrier;
878 BarrierSet *bs = BarrierSet::barrier_set();
879
880 // Argument passing via the stack.
881 const int caller_stack_slots = 3;
882
883 Register R0_pre_val = R0;
884 __ ld(R0, -8, R1_SP);
885 Register R11_tmp1 = R11_scratch1;
886 __ std(R11_tmp1, -16, R1_SP);
887 Register R12_tmp2 = R12_scratch2;
888 __ std(R12_tmp2, -24, R1_SP);
889
890 /* ==== Check whether marking is active ==== */
891 // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
892 // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
893 __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
894
895 __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::MARKING);
896 __ beq(CCR0, skip_barrier);
897
898 /* ==== Add previous value directly to thread-local SATB mark queue ==== */
899 // Check queue's capacity. Jump to runtime if no free slot is available.
900 __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
901 __ cmpdi(CCR0, R12_tmp2, 0);
902 __ beq(CCR0, runtime);
903
904 // Capacity suffices. Decrement the queue's size by one slot (size of one oop).
905 __ addi(R12_tmp2, R12_tmp2, -wordSize);
906 __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
907
908 // Enqueue the previous value and skip the runtime invocation.
909 __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
910 __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
911 __ b(skip_barrier);
912
913 __ bind(runtime);
914
915 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
|
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "gc/shared/gcArguments.hpp"
27 #include "gc/shared/gc_globals.hpp"
28 #include "macroAssembler_ppc.hpp"
29 #include "precompiled.hpp"
30 #include "asm/macroAssembler.inline.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
33 #include "gc/shenandoah/shenandoahForwarding.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahRuntime.hpp"
38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
39 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
40 #include "gc/shenandoah/mode/shenandoahMode.hpp"
41 #include "interpreter/interpreter.hpp"
42 #include "runtime/javaThread.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/globalDefinitions.hpp"
45 #include "vm_version_ppc.hpp"
46 #ifdef COMPILER1
47 #include "c1/c1_LIRAssembler.hpp"
48 #include "c1/c1_MacroAssembler.hpp"
49 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
50 #endif
51
52 #define __ masm->
53
54 void ShenandoahBarrierSetAssembler::satb_write_barrier(MacroAssembler *masm,
55 Register base, RegisterOrConstant ind_or_offs,
56 Register tmp1, Register tmp2, Register tmp3,
57 MacroAssembler::PreservationLevel preservation_level) {
58 if (ShenandoahSATBBarrier) {
59 __ block_comment("satb_write_barrier (shenandoahgc) {");
60 satb_write_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
74 satb_write_barrier_impl(masm, decorators, noreg, noreg, val, tmp1, tmp2, preservation_level);
75 __ block_comment("} iu_barrier (shenandoahgc)");
76 }
77 }
78
79 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
80 Register base, RegisterOrConstant ind_or_offs,
81 Register dst,
82 Register tmp1, Register tmp2,
83 MacroAssembler::PreservationLevel preservation_level) {
84 if (ShenandoahLoadRefBarrier) {
85 __ block_comment("load_reference_barrier (shenandoahgc) {");
86 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
87 __ block_comment("} load_reference_barrier (shenandoahgc)");
88 }
89 }
90
91 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
92 Register src, Register dst, Register count,
93 Register preserve1, Register preserve2) {
94 Register R11_tmp = R11_scratch1;
95
96 assert_different_registers(src, dst, count, R11_tmp, noreg);
97 if (preserve1 != noreg) {
98 // Technically not required, but likely to indicate an error.
99 assert_different_registers(preserve1, preserve2);
100 }
101
102 /* ==== Check whether barrier is required (optimizations) ==== */
103 // Fast path: Component type of array is not a reference type.
104 if (!is_reference_type(type)) {
105 return;
106 }
107
108 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
109
110 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
111 // any useful information.
112 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahIUBarrier && !ShenandoahLoadRefBarrier) {
113 return;
114 }
115
116 __ block_comment("arraycopy_prologue (shenandoahgc) {");
117 Label skip_prologue;
118
119 // Fast path: Array is of length zero.
120 __ cmpdi(CCR0, count, 0);
121 __ beq(CCR0, skip_prologue);
122
123 /* ==== Check whether barrier is required (gc state) ==== */
124 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
125 R16_thread);
126
127 // The set of garbage collection states requiring barriers depends on the available barrier types and the
128 // type of the reference in question.
129 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
130 // for the garbage collector.
131 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
132 ? ShenandoahHeap::HAS_FORWARDED
133 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING;
134
135 __ andi_(R11_tmp, R11_tmp, required_states);
136 __ beq(CCR0, skip_prologue);
137
138 /* ==== Invoke runtime ==== */
139 // Save to-be-preserved registers.
140 int highest_preserve_register_index = 0;
141 {
142 if (preserve1 != noreg && preserve1->is_volatile()) {
143 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
144 }
145 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
146 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
147 }
148
149 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
150 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
151 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
152
153 __ save_LR_CR(R11_tmp);
170 {
171 __ pop_frame();
172 __ restore_LR_CR(R11_tmp);
173
174 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
175 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
176 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
177
178 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
179 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
180 }
181 if (preserve1 != noreg && preserve1->is_volatile()) {
182 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
183 }
184 }
185
186 __ bind(skip_prologue);
187 __ block_comment("} arraycopy_prologue (shenandoahgc)");
188 }
189
190 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
191 Register dst, Register count,
192 Register preserve) {
193 if (is_reference_type(type)) {
194 __ block_comment("arraycopy_epilogue (shenandoahgc) {");
195 gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
196 __ block_comment("} arraycopy_epilogue (shenandoahgc)");
197 }
198 }
199
200 // The to-be-enqueued value can either be determined
201 // - dynamically by passing the reference's address information (load mode) or
202 // - statically by passing a register the value is stored in (preloaded mode)
203 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
204 // - for incremental-update barriers.
205 //
206 // decorators: The previous value's decorator set.
207 // In "load mode", the value must equal '0'.
208 // base: Base register of the reference's address (load mode).
209 // In "preloaded mode", the register must equal 'noreg'.
210 // ind_or_offs: Index or offset of the reference's address (load mode).
211 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
212 // pre_val: Register holding the to-be-stored value (preloaded mode).
213 // In "load mode", this register acts as a temporary register and must
214 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
215 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
216 void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
217 Register base, RegisterOrConstant ind_or_offs,
218 Register pre_val,
219 Register tmp1, Register tmp2,
220 MacroAssembler::PreservationLevel preservation_level) {
221 assert_different_registers(tmp1, tmp2, pre_val, noreg);
222
223 Label skip_barrier;
224
225 /* ==== Determine necessary runtime invocation preservation measures ==== */
226 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
227 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
228 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
229
230 // Check whether marking is active.
231 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
232
233 __ andi_(tmp1, tmp1, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
234 __ beq(CCR0, skip_barrier);
235
236 /* ==== Determine the reference's previous value ==== */
237 bool preloaded_mode = base == noreg;
238 Register pre_val_save = noreg;
239
240 if (preloaded_mode) {
241 // Previous value has been passed to the method, so it must not be determined manually.
242 // In case 'pre_val' is a volatile register, it must be saved across the C-call
243 // as callers may depend on its value.
244 // Unless the general purposes registers are saved anyway, one of the temporary registers
245 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
246 if (!preserve_gp_registers && pre_val->is_volatile()) {
247 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
248 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
249 }
250
251 if ((decorators & IS_NOT_NULL) != 0) {
252 #ifdef ASSERT
253 __ cmpdi(CCR0, pre_val, 0);
579 base, ind_or_offs,
580 intermediate_dst,
581 tmp1, R0,
582 preservation_level);
583
584 __ mr_if_needed(dst, intermediate_dst);
585 } else {
586 BarrierSetAssembler::load_at(masm, decorators, type,
587 base, ind_or_offs,
588 dst,
589 tmp1, tmp2,
590 preservation_level, L_handle_null);
591 }
592
593 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
594 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
595 iu_barrier(masm, dst, tmp1, tmp2, preservation_level);
596 }
597 }
598
599 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
600 if (!ShenandoahHeap::heap()->mode()->is_generational()) {
601 return;
602 }
603
604 ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
605 CardTable* ct = ctbs->card_table();
606 assert_different_registers(base, tmp, R0);
607
608 if (ind_or_offs.is_constant()) {
609 __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
610 } else {
611 __ add(base, ind_or_offs.as_register(), base);
612 }
613
614 __ load_const_optimized(tmp, (address)ct->byte_map_base(), R0);
615 __ srdi(base, base, CardTable::card_shift());
616 __ li(R0, CardTable::dirty_card_val());
617 __ stbx(R0, tmp, base);
618 }
619
620 // base: Base register of the reference's address.
621 // ind_or_offs: Index or offset of the reference's address.
622 // val: To-be-stored value/reference's new value.
623 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
624 Register base, RegisterOrConstant ind_or_offs, Register val,
625 Register tmp1, Register tmp2, Register tmp3,
626 MacroAssembler::PreservationLevel preservation_level) {
627 if (is_reference_type(type)) {
628 if (ShenandoahSATBBarrier) {
629 satb_write_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
630 }
631
632 if (ShenandoahIUBarrier && val != noreg) {
633 iu_barrier(masm, val, tmp1, tmp2, preservation_level, decorators);
634 }
635 }
636
637 BarrierSetAssembler::store_at(masm, decorators, type,
638 base, ind_or_offs,
639 val,
640 tmp1, tmp2, tmp3,
641 preservation_level);
642
643 // No need for post barrier if storing NULL
644 if (is_reference_type(type) && val != noreg) {
645 store_check(masm, base, ind_or_offs, tmp1);
646 }
647 }
648
649 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
650 Register dst, Register jni_env, Register obj,
651 Register tmp, Label &slowpath) {
652 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
653
654 assert_different_registers(jni_env, obj, tmp);
655
656 Label done;
657
658 // Fast path: Reference is null (JNI tags are zero for null pointers).
659 __ cmpdi(CCR0, obj, 0);
660 __ beq(CCR0, done);
661
662 // Resolve jobject using standard implementation.
663 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
664
665 // Check whether heap is stable.
666 __ lbz(tmp,
776 // unlikely case should be handled by the smallest possible code. Instead of emitting a third,
777 // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
778 // (passed arguments are identical).
779 //
780 // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
781 // to fail. Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
782 // It is thus important to ensure that a re-execution of those steps does not put program correctness
783 // at risk:
784 // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
785 // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
786 // fails. Unless the reference has been updated in the meanwhile once again, this is
787 // guaranteed to be the case.
788 // In case of a concurrent update, the CAS would be retried again. This is legitimate
789 // in terms of program correctness (even though it is not desired).
790 __ bne(CCR0, step_four);
791
792 __ bind(done);
793 __ block_comment("} cmpxchg_oop (shenandoahgc)");
794 }
795
796 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
797 Register addr, Register count, Register preserve) {
798 if (!ShenandoahHeap::heap()->mode()->is_generational()) {
799 return;
800 }
801
802 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
803 CardTable* ct = bs->card_table();
804 assert_different_registers(addr, count, R0);
805
806 Label Lskip_loop, Lstore_loop;
807
808 __ sldi_(count, count, LogBytesPerHeapOop);
809 __ beq(CCR0, Lskip_loop); // zero length
810 __ addi(count, count, -BytesPerHeapOop);
811 __ add(count, addr, count);
812 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
813 __ srdi(addr, addr, CardTable::card_shift());
814 __ srdi(count, count, CardTable::card_shift());
815 __ subf(count, addr, count);
816 __ add_const_optimized(addr, addr, (address)ct->byte_map_base(), R0);
817 __ addi(count, count, 1);
818 __ li(R0, 0);
819 __ mtctr(count);
820 // Byte store loop
821 __ bind(Lstore_loop);
822 __ stb(R0, 0, addr);
823 __ addi(addr, addr, 1);
824 __ bdnz(Lstore_loop);
825 __ bind(Lskip_loop);
826 }
827
828 #undef __
829
830 #ifdef COMPILER1
831
832 #define __ ce->masm()->
833
834 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
835 __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
836
837 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
838 __ bind(*stub->entry());
839
840 // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
841 // This stub is the slowpath of that function.
842
843 assert(stub->pre_val()->is_register(), "pre_val must be a register");
844 Register pre_val = stub->pre_val()->as_register();
845
846 // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
847 // ("preloaded mode" of the store barrier).
943 __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
944
945 Label runtime, skip_barrier;
946 BarrierSet *bs = BarrierSet::barrier_set();
947
948 // Argument passing via the stack.
949 const int caller_stack_slots = 3;
950
951 Register R0_pre_val = R0;
952 __ ld(R0, -8, R1_SP);
953 Register R11_tmp1 = R11_scratch1;
954 __ std(R11_tmp1, -16, R1_SP);
955 Register R12_tmp2 = R12_scratch2;
956 __ std(R12_tmp2, -24, R1_SP);
957
958 /* ==== Check whether marking is active ==== */
959 // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
960 // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
961 __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
962
963 __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
964 __ beq(CCR0, skip_barrier);
965
966 /* ==== Add previous value directly to thread-local SATB mark queue ==== */
967 // Check queue's capacity. Jump to runtime if no free slot is available.
968 __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
969 __ cmpdi(CCR0, R12_tmp2, 0);
970 __ beq(CCR0, runtime);
971
972 // Capacity suffices. Decrement the queue's size by one slot (size of one oop).
973 __ addi(R12_tmp2, R12_tmp2, -wordSize);
974 __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
975
976 // Enqueue the previous value and skip the runtime invocation.
977 __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
978 __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
979 __ b(skip_barrier);
980
981 __ bind(runtime);
982
983 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
|