20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "gc/shared/gcArguments.hpp"
27 #include "gc/shared/gc_globals.hpp"
28 #include "macroAssembler_ppc.hpp"
29 #include "precompiled.hpp"
30 #include "asm/macroAssembler.inline.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
33 #include "gc/shenandoah/shenandoahForwarding.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahRuntime.hpp"
38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
39 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "utilities/globalDefinitions.hpp"
44 #include "vm_version_ppc.hpp"
45
46 #ifdef COMPILER1
47
48 #include "c1/c1_LIRAssembler.hpp"
49 #include "c1/c1_MacroAssembler.hpp"
50 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
51
52 #endif
53
54 #define __ masm->
55
56 void ShenandoahBarrierSetAssembler::satb_write_barrier(MacroAssembler *masm,
57 Register base, RegisterOrConstant ind_or_offs,
58 Register tmp1, Register tmp2, Register tmp3,
59 MacroAssembler::PreservationLevel preservation_level) {
76 satb_write_barrier_impl(masm, decorators, noreg, noreg, val, tmp1, tmp2, preservation_level);
77 __ block_comment("} iu_barrier (shenandoahgc)");
78 }
79 }
80
81 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
82 Register base, RegisterOrConstant ind_or_offs,
83 Register dst,
84 Register tmp1, Register tmp2,
85 MacroAssembler::PreservationLevel preservation_level) {
86 if (ShenandoahLoadRefBarrier) {
87 __ block_comment("load_reference_barrier (shenandoahgc) {");
88 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
89 __ block_comment("} load_reference_barrier (shenandoahgc)");
90 }
91 }
92
93 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
94 Register src, Register dst, Register count,
95 Register preserve1, Register preserve2) {
96 __ block_comment("arraycopy_prologue (shenandoahgc) {");
97
98 Register R11_tmp = R11_scratch1;
99
100 assert_different_registers(src, dst, count, R11_tmp, noreg);
101 if (preserve1 != noreg) {
102 // Technically not required, but likely to indicate an error.
103 assert_different_registers(preserve1, preserve2);
104 }
105
106 /* ==== Check whether barrier is required (optimizations) ==== */
107 // Fast path: Component type of array is not a reference type.
108 if (!is_reference_type(type)) {
109 return;
110 }
111
112 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
113
114 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
115 // any useful information.
116 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahIUBarrier && !ShenandoahLoadRefBarrier) {
117 return;
118 }
119
120 Label skip_prologue;
121
122 // Fast path: Array is of length zero.
123 __ cmpdi(CCR0, count, 0);
124 __ beq(CCR0, skip_prologue);
125
126 /* ==== Check whether barrier is required (gc state) ==== */
127 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
128 R16_thread);
129
130 // The set of garbage collection states requiring barriers depends on the available barrier types and the
131 // type of the reference in question.
132 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
133 // for the garbage collector.
134 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
135 ? ShenandoahHeap::HAS_FORWARDED
136 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
137
138 __ andi_(R11_tmp, R11_tmp, required_states);
139 __ beq(CCR0, skip_prologue);
140
141 /* ==== Invoke runtime ==== */
142 // Save to-be-preserved registers.
143 int highest_preserve_register_index = 0;
144 {
145 if (preserve1 != noreg && preserve1->is_volatile()) {
146 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
147 }
148 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
149 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
150 }
151
152 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
153 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
154 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
155
156 __ save_LR_CR(R11_tmp);
173 {
174 __ pop_frame();
175 __ restore_LR_CR(R11_tmp);
176
177 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
178 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
179 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
180
181 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
182 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
183 }
184 if (preserve1 != noreg && preserve1->is_volatile()) {
185 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
186 }
187 }
188
189 __ bind(skip_prologue);
190 __ block_comment("} arraycopy_prologue (shenandoahgc)");
191 }
192
193 // The to-be-enqueued value can either be determined
194 // - dynamically by passing the reference's address information (load mode) or
195 // - statically by passing a register the value is stored in (preloaded mode)
196 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
197 // - for incremental-update barriers.
198 //
199 // decorators: The previous value's decorator set.
200 // In "load mode", the value must equal '0'.
201 // base: Base register of the reference's address (load mode).
202 // In "preloaded mode", the register must equal 'noreg'.
203 // ind_or_offs: Index or offset of the reference's address (load mode).
204 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
205 // pre_val: Register holding the to-be-stored value (preloaded mode).
206 // In "load mode", this register acts as a temporary register and must
207 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
208 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
209 void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
210 Register base, RegisterOrConstant ind_or_offs,
211 Register pre_val,
212 Register tmp1, Register tmp2,
213 MacroAssembler::PreservationLevel preservation_level) {
214 assert_different_registers(tmp1, tmp2, pre_val, noreg);
215
216 Label skip_barrier;
217
218 /* ==== Determine necessary runtime invocation preservation measures ==== */
219 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
220 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
221 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
222
223 // Check whether marking is active.
224 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
225
226 __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
227 __ beq(CCR0, skip_barrier);
228
229 /* ==== Determine the reference's previous value ==== */
230 bool preloaded_mode = base == noreg;
231 Register pre_val_save = noreg;
232
233 if (preloaded_mode) {
234 // Previous value has been passed to the method, so it must not be determined manually.
235 // In case 'pre_val' is a volatile register, it must be saved across the C-call
236 // as callers may depend on its value.
237 // Unless the general purposes registers are saved anyway, one of the temporary registers
238 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
239 if (!preserve_gp_registers && pre_val->is_volatile()) {
240 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
241 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
242 }
243
244 if ((decorators & IS_NOT_NULL) != 0) {
245 #ifdef ASSERT
246 __ cmpdi(CCR0, pre_val, 0);
572 base, ind_or_offs,
573 intermediate_dst,
574 tmp1, R0,
575 preservation_level);
576
577 __ mr_if_needed(dst, intermediate_dst);
578 } else {
579 BarrierSetAssembler::load_at(masm, decorators, type,
580 base, ind_or_offs,
581 dst,
582 tmp1, tmp2,
583 preservation_level, L_handle_null);
584 }
585
586 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
587 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
588 iu_barrier(masm, dst, tmp1, tmp2, preservation_level);
589 }
590 }
591
592 // base: Base register of the reference's address.
593 // ind_or_offs: Index or offset of the reference's address.
594 // val: To-be-stored value/reference's new value.
595 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
596 Register base, RegisterOrConstant ind_or_offs, Register val,
597 Register tmp1, Register tmp2, Register tmp3,
598 MacroAssembler::PreservationLevel preservation_level) {
599 if (is_reference_type(type)) {
600 if (ShenandoahSATBBarrier) {
601 satb_write_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
602 }
603
604 if (ShenandoahIUBarrier && val != noreg) {
605 iu_barrier(masm, val, tmp1, tmp2, preservation_level, decorators);
606 }
607 }
608
609 BarrierSetAssembler::store_at(masm, decorators, type,
610 base, ind_or_offs,
611 val,
612 tmp1, tmp2, tmp3,
613 preservation_level);
614 }
615
616 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
617 Register dst, Register jni_env, Register obj,
618 Register tmp, Label &slowpath) {
619 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
620
621 assert_different_registers(jni_env, obj, tmp);
622
623 Label done;
624
625 // Fast path: Reference is null (JNI tags are zero for null pointers).
626 __ cmpdi(CCR0, obj, 0);
627 __ beq(CCR0, done);
628
629 // Resolve jobject using standard implementation.
630 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
631
632 // Check whether heap is stable.
633 __ lbz(tmp,
743 // unlikely case should be handled by the smallest possible code. Instead of emitting a third,
744 // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
745 // (passed arguments are identical).
746 //
747 // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
748 // to fail. Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
749 // It is thus important to ensure that a re-execution of those steps does not put program correctness
750 // at risk:
751 // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
752 // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
753 // fails. Unless the reference has been updated in the meanwhile once again, this is
754 // guaranteed to be the case.
755 // In case of a concurrent update, the CAS would be retried again. This is legitimate
756 // in terms of program correctness (even though it is not desired).
757 __ bne(CCR0, step_four);
758
759 __ bind(done);
760 __ block_comment("} cmpxchg_oop (shenandoahgc)");
761 }
762
763 #undef __
764
765 #ifdef COMPILER1
766
767 #define __ ce->masm()->
768
769 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
770 __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
771
772 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
773 __ bind(*stub->entry());
774
775 // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
776 // This stub is the slowpath of that function.
777
778 assert(stub->pre_val()->is_register(), "pre_val must be a register");
779 Register pre_val = stub->pre_val()->as_register();
780
781 // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
782 // ("preloaded mode" of the store barrier).
878 __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
879
880 Label runtime, skip_barrier;
881 BarrierSet *bs = BarrierSet::barrier_set();
882
883 // Argument passing via the stack.
884 const int caller_stack_slots = 3;
885
886 Register R0_pre_val = R0;
887 __ ld(R0, -8, R1_SP);
888 Register R11_tmp1 = R11_scratch1;
889 __ std(R11_tmp1, -16, R1_SP);
890 Register R12_tmp2 = R12_scratch2;
891 __ std(R12_tmp2, -24, R1_SP);
892
893 /* ==== Check whether marking is active ==== */
894 // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
895 // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
896 __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
897
898 __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::MARKING);
899 __ beq(CCR0, skip_barrier);
900
901 /* ==== Add previous value directly to thread-local SATB mark queue ==== */
902 // Check queue's capacity. Jump to runtime if no free slot is available.
903 __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
904 __ cmpdi(CCR0, R12_tmp2, 0);
905 __ beq(CCR0, runtime);
906
907 // Capacity suffices. Decrement the queue's size by one slot (size of one oop).
908 __ addi(R12_tmp2, R12_tmp2, -wordSize);
909 __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
910
911 // Enqueue the previous value and skip the runtime invocation.
912 __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
913 __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
914 __ b(skip_barrier);
915
916 __ bind(runtime);
917
918 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
|
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "gc/shared/gcArguments.hpp"
27 #include "gc/shared/gc_globals.hpp"
28 #include "macroAssembler_ppc.hpp"
29 #include "precompiled.hpp"
30 #include "asm/macroAssembler.inline.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
33 #include "gc/shenandoah/shenandoahForwarding.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahRuntime.hpp"
38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
39 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
40 #include "gc/shenandoah/mode/shenandoahMode.hpp"
41 #include "interpreter/interpreter.hpp"
42 #include "runtime/javaThread.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/globalDefinitions.hpp"
45 #include "vm_version_ppc.hpp"
46
47 #ifdef COMPILER1
48
49 #include "c1/c1_LIRAssembler.hpp"
50 #include "c1/c1_MacroAssembler.hpp"
51 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
52
53 #endif
54
55 #define __ masm->
56
57 void ShenandoahBarrierSetAssembler::satb_write_barrier(MacroAssembler *masm,
58 Register base, RegisterOrConstant ind_or_offs,
59 Register tmp1, Register tmp2, Register tmp3,
60 MacroAssembler::PreservationLevel preservation_level) {
77 satb_write_barrier_impl(masm, decorators, noreg, noreg, val, tmp1, tmp2, preservation_level);
78 __ block_comment("} iu_barrier (shenandoahgc)");
79 }
80 }
81
82 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
83 Register base, RegisterOrConstant ind_or_offs,
84 Register dst,
85 Register tmp1, Register tmp2,
86 MacroAssembler::PreservationLevel preservation_level) {
87 if (ShenandoahLoadRefBarrier) {
88 __ block_comment("load_reference_barrier (shenandoahgc) {");
89 load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
90 __ block_comment("} load_reference_barrier (shenandoahgc)");
91 }
92 }
93
94 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
95 Register src, Register dst, Register count,
96 Register preserve1, Register preserve2) {
97 Register R11_tmp = R11_scratch1;
98
99 assert_different_registers(src, dst, count, R11_tmp, noreg);
100 if (preserve1 != noreg) {
101 // Technically not required, but likely to indicate an error.
102 assert_different_registers(preserve1, preserve2);
103 }
104
105 /* ==== Check whether barrier is required (optimizations) ==== */
106 // Fast path: Component type of array is not a reference type.
107 if (!is_reference_type(type)) {
108 return;
109 }
110
111 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
112
113 // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
114 // any useful information.
115 if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahIUBarrier && !ShenandoahLoadRefBarrier) {
116 return;
117 }
118
119 __ block_comment("arraycopy_prologue (shenandoahgc) {");
120 Label skip_prologue;
121
122 // Fast path: Array is of length zero.
123 __ cmpdi(CCR0, count, 0);
124 __ beq(CCR0, skip_prologue);
125
126 /* ==== Check whether barrier is required (gc state) ==== */
127 __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
128 R16_thread);
129
130 // The set of garbage collection states requiring barriers depends on the available barrier types and the
131 // type of the reference in question.
132 // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
133 // for the garbage collector.
134 const int required_states = ShenandoahSATBBarrier && dest_uninitialized
135 ? ShenandoahHeap::HAS_FORWARDED
136 : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING;
137
138 __ andi_(R11_tmp, R11_tmp, required_states);
139 __ beq(CCR0, skip_prologue);
140
141 /* ==== Invoke runtime ==== */
142 // Save to-be-preserved registers.
143 int highest_preserve_register_index = 0;
144 {
145 if (preserve1 != noreg && preserve1->is_volatile()) {
146 __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
147 }
148 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
149 __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
150 }
151
152 __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
153 __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
154 __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
155
156 __ save_LR_CR(R11_tmp);
173 {
174 __ pop_frame();
175 __ restore_LR_CR(R11_tmp);
176
177 __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
178 __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
179 __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
180
181 if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
182 __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
183 }
184 if (preserve1 != noreg && preserve1->is_volatile()) {
185 __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
186 }
187 }
188
189 __ bind(skip_prologue);
190 __ block_comment("} arraycopy_prologue (shenandoahgc)");
191 }
192
193 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
194 Register dst, Register count,
195 Register preserve) {
196 if (is_reference_type(type)) {
197 __ block_comment("arraycopy_epilogue (shenandoahgc) {");
198 gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
199 __ block_comment("} arraycopy_epilogue (shenandoahgc)");
200 }
201 }
202
203 // The to-be-enqueued value can either be determined
204 // - dynamically by passing the reference's address information (load mode) or
205 // - statically by passing a register the value is stored in (preloaded mode)
206 // - for performance optimizations in cases where the previous value is known (currently not implemented) and
207 // - for incremental-update barriers.
208 //
209 // decorators: The previous value's decorator set.
210 // In "load mode", the value must equal '0'.
211 // base: Base register of the reference's address (load mode).
212 // In "preloaded mode", the register must equal 'noreg'.
213 // ind_or_offs: Index or offset of the reference's address (load mode).
214 // If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
215 // pre_val: Register holding the to-be-stored value (preloaded mode).
216 // In "load mode", this register acts as a temporary register and must
217 // thus not be 'noreg'. In "preloaded mode", its content will be sustained.
218 // tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
219 void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
220 Register base, RegisterOrConstant ind_or_offs,
221 Register pre_val,
222 Register tmp1, Register tmp2,
223 MacroAssembler::PreservationLevel preservation_level) {
224 assert_different_registers(tmp1, tmp2, pre_val, noreg);
225
226 Label skip_barrier;
227
228 /* ==== Determine necessary runtime invocation preservation measures ==== */
229 const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
230 const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
231 const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
232
233 // Check whether marking is active.
234 __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
235
236 __ andi_(tmp1, tmp1, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
237 __ beq(CCR0, skip_barrier);
238
239 /* ==== Determine the reference's previous value ==== */
240 bool preloaded_mode = base == noreg;
241 Register pre_val_save = noreg;
242
243 if (preloaded_mode) {
244 // Previous value has been passed to the method, so it must not be determined manually.
245 // In case 'pre_val' is a volatile register, it must be saved across the C-call
246 // as callers may depend on its value.
247 // Unless the general purposes registers are saved anyway, one of the temporary registers
248 // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
249 if (!preserve_gp_registers && pre_val->is_volatile()) {
250 pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
251 assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
252 }
253
254 if ((decorators & IS_NOT_NULL) != 0) {
255 #ifdef ASSERT
256 __ cmpdi(CCR0, pre_val, 0);
582 base, ind_or_offs,
583 intermediate_dst,
584 tmp1, R0,
585 preservation_level);
586
587 __ mr_if_needed(dst, intermediate_dst);
588 } else {
589 BarrierSetAssembler::load_at(masm, decorators, type,
590 base, ind_or_offs,
591 dst,
592 tmp1, tmp2,
593 preservation_level, L_handle_null);
594 }
595
596 /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
597 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
598 iu_barrier(masm, dst, tmp1, tmp2, preservation_level);
599 }
600 }
601
602 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
603 if (!ShenandoahHeap::heap()->mode()->is_generational()) {
604 return;
605 }
606
607 ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
608 CardTable* ct = ctbs->card_table();
609 assert_different_registers(base, tmp, R0);
610
611 if (ind_or_offs.is_constant()) {
612 __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
613 } else {
614 __ add(base, ind_or_offs.as_register(), base);
615 }
616
617 __ load_const_optimized(tmp, (address)ct->byte_map_base(), R0);
618 __ srdi(base, base, CardTable::card_shift());
619 __ li(R0, CardTable::dirty_card_val());
620 __ stbx(R0, tmp, base);
621 }
622
623 // base: Base register of the reference's address.
624 // ind_or_offs: Index or offset of the reference's address.
625 // val: To-be-stored value/reference's new value.
626 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
627 Register base, RegisterOrConstant ind_or_offs, Register val,
628 Register tmp1, Register tmp2, Register tmp3,
629 MacroAssembler::PreservationLevel preservation_level) {
630 if (is_reference_type(type)) {
631 if (ShenandoahSATBBarrier) {
632 satb_write_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
633 }
634
635 if (ShenandoahIUBarrier && val != noreg) {
636 iu_barrier(masm, val, tmp1, tmp2, preservation_level, decorators);
637 }
638 }
639
640 BarrierSetAssembler::store_at(masm, decorators, type,
641 base, ind_or_offs,
642 val,
643 tmp1, tmp2, tmp3,
644 preservation_level);
645
646 // No need for post barrier if storing NULL
647 if (is_reference_type(type) && val != noreg) {
648 store_check(masm, base, ind_or_offs, tmp1);
649 }
650 }
651
652 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
653 Register dst, Register jni_env, Register obj,
654 Register tmp, Label &slowpath) {
655 __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
656
657 assert_different_registers(jni_env, obj, tmp);
658
659 Label done;
660
661 // Fast path: Reference is null (JNI tags are zero for null pointers).
662 __ cmpdi(CCR0, obj, 0);
663 __ beq(CCR0, done);
664
665 // Resolve jobject using standard implementation.
666 BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
667
668 // Check whether heap is stable.
669 __ lbz(tmp,
779 // unlikely case should be handled by the smallest possible code. Instead of emitting a third,
780 // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
781 // (passed arguments are identical).
782 //
783 // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
784 // to fail. Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
785 // It is thus important to ensure that a re-execution of those steps does not put program correctness
786 // at risk:
787 // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
788 // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
789 // fails. Unless the reference has been updated in the meanwhile once again, this is
790 // guaranteed to be the case.
791 // In case of a concurrent update, the CAS would be retried again. This is legitimate
792 // in terms of program correctness (even though it is not desired).
793 __ bne(CCR0, step_four);
794
795 __ bind(done);
796 __ block_comment("} cmpxchg_oop (shenandoahgc)");
797 }
798
799 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
800 Register addr, Register count, Register preserve) {
801 if (!ShenandoahHeap::heap()->mode()->is_generational()) {
802 return;
803 }
804
805 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
806 CardTable* ct = bs->card_table();
807 assert_different_registers(addr, count, R0);
808
809 Label Lskip_loop, Lstore_loop;
810
811 __ sldi_(count, count, LogBytesPerHeapOop);
812 __ beq(CCR0, Lskip_loop); // zero length
813 __ addi(count, count, -BytesPerHeapOop);
814 __ add(count, addr, count);
815 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
816 __ srdi(addr, addr, CardTable::card_shift());
817 __ srdi(count, count, CardTable::card_shift());
818 __ subf(count, addr, count);
819 __ add_const_optimized(addr, addr, (address)ct->byte_map_base(), R0);
820 __ addi(count, count, 1);
821 __ li(R0, 0);
822 __ mtctr(count);
823 // Byte store loop
824 __ bind(Lstore_loop);
825 __ stb(R0, 0, addr);
826 __ addi(addr, addr, 1);
827 __ bdnz(Lstore_loop);
828 __ bind(Lskip_loop);
829 }
830
831 #undef __
832
833 #ifdef COMPILER1
834
835 #define __ ce->masm()->
836
837 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
838 __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
839
840 ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
841 __ bind(*stub->entry());
842
843 // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
844 // This stub is the slowpath of that function.
845
846 assert(stub->pre_val()->is_register(), "pre_val must be a register");
847 Register pre_val = stub->pre_val()->as_register();
848
849 // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
850 // ("preloaded mode" of the store barrier).
946 __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
947
948 Label runtime, skip_barrier;
949 BarrierSet *bs = BarrierSet::barrier_set();
950
951 // Argument passing via the stack.
952 const int caller_stack_slots = 3;
953
954 Register R0_pre_val = R0;
955 __ ld(R0, -8, R1_SP);
956 Register R11_tmp1 = R11_scratch1;
957 __ std(R11_tmp1, -16, R1_SP);
958 Register R12_tmp2 = R12_scratch2;
959 __ std(R12_tmp2, -24, R1_SP);
960
961 /* ==== Check whether marking is active ==== */
962 // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
963 // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
964 __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
965
966 __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
967 __ beq(CCR0, skip_barrier);
968
969 /* ==== Add previous value directly to thread-local SATB mark queue ==== */
970 // Check queue's capacity. Jump to runtime if no free slot is available.
971 __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
972 __ cmpdi(CCR0, R12_tmp2, 0);
973 __ beq(CCR0, runtime);
974
975 // Capacity suffices. Decrement the queue's size by one slot (size of one oop).
976 __ addi(R12_tmp2, R12_tmp2, -wordSize);
977 __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
978
979 // Enqueue the previous value and skip the runtime invocation.
980 __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
981 __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
982 __ b(skip_barrier);
983
984 __ bind(runtime);
985
986 /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
|