14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahForwarding.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "runtime/javaThread.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "utilities/macros.hpp"
38 #ifdef COMPILER1
39 #include "c1/c1_LIRAssembler.hpp"
40 #include "c1/c1_MacroAssembler.hpp"
41 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
42 #endif
43
44 #define __ masm->
45
46 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
47 if (handle_gpr) {
48 __ push_IU_state();
49 }
50
51 if (handle_fp) {
52 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
53 LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));
103 __ movflt(xmm5, Address(rsp, xmm_size * 5));
104 __ movflt(xmm6, Address(rsp, xmm_size * 6));
105 __ movflt(xmm7, Address(rsp, xmm_size * 7));
106 __ addptr(rsp, xmm_size * 8);
107 } else {
108 __ pop_FPU_state();
109 }
110 }
111
112 if (handle_gpr) {
113 __ pop_IU_state();
114 }
115 }
116
117 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
118 Register src, Register dst, Register count) {
119
120 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
121
122 if (is_reference_type(type)) {
123
124 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
125 #ifdef _LP64
126 Register thread = r15_thread;
127 #else
128 Register thread = rax;
129 if (thread == src || thread == dst || thread == count) {
130 thread = rbx;
131 }
132 if (thread == src || thread == dst || thread == count) {
133 thread = rcx;
134 }
135 if (thread == src || thread == dst || thread == count) {
136 thread = rdx;
137 }
138 __ push(thread);
139 __ get_thread(thread);
140 #endif
141 assert_different_registers(src, dst, count, thread);
142
143 Label done;
144 // Short-circuit if count == 0.
145 __ testptr(count, count);
146 __ jcc(Assembler::zero, done);
147
148 // Avoid runtime call when not active.
149 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
150 int flags;
151 if (ShenandoahSATBBarrier && dest_uninitialized) {
152 flags = ShenandoahHeap::HAS_FORWARDED;
153 } else {
154 flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
155 }
156 __ testb(gc_state, flags);
157 __ jcc(Assembler::zero, done);
158
159 save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
160
161 #ifdef _LP64
162 assert(src == rdi, "expected");
163 assert(dst == rsi, "expected");
164 assert(count == rdx, "expected");
165 if (UseCompressedOops) {
166 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
167 src, dst, count);
168 } else
169 #endif
170 {
171 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
172 src, dst, count);
173 }
174
175 restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
176
177 __ bind(done);
178 NOT_LP64(__ pop(thread);)
179 }
180 }
181
182 }
183
184 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
185 Register obj,
186 Register pre_val,
187 Register thread,
188 Register tmp,
189 bool tosca_live,
190 bool expand_call) {
191
192 if (ShenandoahSATBBarrier) {
193 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
194 }
195 }
196
197 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
198 Register obj,
199 Register pre_val,
200 Register thread,
201 Register tmp,
202 bool tosca_live,
203 bool expand_call) {
207
208 #ifdef _LP64
209 assert(thread == r15_thread, "must be");
210 #endif // _LP64
211
212 Label done;
213 Label runtime;
214
215 assert(pre_val != noreg, "check this code");
216
217 if (obj != noreg) {
218 assert_different_registers(obj, pre_val, tmp);
219 assert(pre_val != rax, "check this code");
220 }
221
222 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
223 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
224 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
225
226 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
227 __ testb(gc_state, ShenandoahHeap::MARKING);
228 __ jcc(Assembler::zero, done);
229
230 // Do we need to load the previous value?
231 if (obj != noreg) {
232 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
233 }
234
235 // Is the previous value null?
236 __ cmpptr(pre_val, NULL_WORD);
237 __ jcc(Assembler::equal, done);
238
239 // Can we store original value in the thread's buffer?
240 // Is index == 0?
241 // (The index field is typed as size_t.)
242
243 __ movptr(tmp, index); // tmp := *index_adr
244 __ cmpptr(tmp, 0); // tmp == 0?
245 __ jcc(Assembler::equal, runtime); // If yes, goto runtime
246
247 __ subptr(tmp, wordSize); // tmp := tmp - wordSize
573 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
574 assert_different_registers(dst, tmp1, tmp_thread);
575 if (!thread->is_valid()) {
576 thread = rdx;
577 }
578 NOT_LP64(__ get_thread(thread));
579 // Generate the SATB pre-barrier code to log the value of
580 // the referent field in an SATB buffer.
581 shenandoah_write_barrier_pre(masm /* masm */,
582 noreg /* obj */,
583 dst /* pre_val */,
584 thread /* thread */,
585 tmp1 /* tmp */,
586 true /* tosca_live */,
587 true /* expand_call */);
588
589 restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
590 }
591 }
592
593 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
594 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
595
596 bool on_oop = is_reference_type(type);
597 bool in_heap = (decorators & IN_HEAP) != 0;
598 bool as_normal = (decorators & AS_NORMAL) != 0;
599 if (on_oop && in_heap) {
600 bool needs_pre_barrier = as_normal;
601
602 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
603 // flatten object address if needed
604 // We do it regardless of precise because we need the registers
605 if (dst.index() == noreg && dst.disp() == 0) {
606 if (dst.base() != tmp1) {
607 __ movptr(tmp1, dst.base());
608 }
609 } else {
610 __ lea(tmp1, dst);
611 }
612
614
615 #ifndef _LP64
616 __ get_thread(rthread);
617 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
618 imasm->save_bcp();
619 #endif
620
621 if (needs_pre_barrier) {
622 shenandoah_write_barrier_pre(masm /*masm*/,
623 tmp1 /* obj */,
624 tmp2 /* pre_val */,
625 rthread /* thread */,
626 tmp3 /* tmp */,
627 val != noreg /* tosca_live */,
628 false /* expand_call */);
629 }
630 if (val == noreg) {
631 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
632 } else {
633 iu_barrier(masm, val, tmp3);
634 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
635 }
636 NOT_LP64(imasm->restore_bcp());
637 } else {
638 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
639 }
640 }
641
642 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
643 Register obj, Register tmp, Label& slowpath) {
644 Label done;
645 // Resolve jobject
646 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
647
648 // Check for null.
649 __ testptr(obj, obj);
650 __ jcc(Assembler::zero, done);
651
652 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
653 __ testb(gc_state, ShenandoahHeap::EVACUATION);
654 __ jccb(Assembler::notZero, slowpath);
810 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
811 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
812
813 if (exchange) {
814 __ bind(L_failure);
815 __ bind(L_success);
816 } else {
817 assert(res != noreg, "need result register");
818
819 Label exit;
820 __ bind(L_failure);
821 __ xorptr(res, res);
822 __ jmpb(exit);
823
824 __ bind(L_success);
825 __ movptr(res, 1);
826 __ bind(exit);
827 }
828 }
829
830 #undef __
831
832 #ifdef COMPILER1
833
834 #define __ ce->masm()->
835
836 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
837 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
838 // At this point we know that marking is in progress.
839 // If do_load() is true then we have to emit the
840 // load of the previous value; otherwise it has already
841 // been loaded into _pre_val.
842
843 __ bind(*stub->entry());
844 assert(stub->pre_val()->is_register(), "Precondition.");
845
846 Register pre_val_reg = stub->pre_val()->as_register();
847
848 if (stub->do_load()) {
849 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
925 __ prologue("shenandoah_pre_barrier", false);
926 // arg0 : previous value of memory
927
928 __ push(rax);
929 __ push(rdx);
930
931 const Register pre_val = rax;
932 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
933 const Register tmp = rdx;
934
935 NOT_LP64(__ get_thread(thread);)
936
937 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
938 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
939
940 Label done;
941 Label runtime;
942
943 // Is SATB still active?
944 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
945 __ testb(gc_state, ShenandoahHeap::MARKING);
946 __ jcc(Assembler::zero, done);
947
948 // Can we store original value in the thread's buffer?
949
950 __ movptr(tmp, queue_index);
951 __ testptr(tmp, tmp);
952 __ jcc(Assembler::zero, runtime);
953 __ subptr(tmp, wordSize);
954 __ movptr(queue_index, tmp);
955 __ addptr(tmp, buffer);
956
957 // prev_val (rax)
958 __ load_parameter(0, pre_val);
959 __ movptr(Address(tmp, 0), pre_val);
960 __ jmp(done);
961
962 __ bind(runtime);
963
964 __ save_live_registers_no_oop_map(true);
965
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahForwarding.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
34 #include "gc/shenandoah/mode/shenandoahMode.hpp"
35 #include "interpreter/interpreter.hpp"
36 #include "runtime/javaThread.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "utilities/macros.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
48 if (handle_gpr) {
49 __ push_IU_state();
50 }
51
52 if (handle_fp) {
53 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
54 LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));
104 __ movflt(xmm5, Address(rsp, xmm_size * 5));
105 __ movflt(xmm6, Address(rsp, xmm_size * 6));
106 __ movflt(xmm7, Address(rsp, xmm_size * 7));
107 __ addptr(rsp, xmm_size * 8);
108 } else {
109 __ pop_FPU_state();
110 }
111 }
112
113 if (handle_gpr) {
114 __ pop_IU_state();
115 }
116 }
117
118 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
119 Register src, Register dst, Register count) {
120
121 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
122
123 if (is_reference_type(type)) {
124 if (ShenandoahHeap::heap()->mode()->is_generational()) {
125 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
126 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
127 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
128
129 // We need to squirrel away the original element count because the
130 // array copy assembly will destroy the value and we need it for the
131 // card marking barrier.
132 #ifdef _LP64
133 if (!checkcast) {
134 if (!obj_int) {
135 // Save count for barrier
136 __ movptr(r11, count);
137 } else if (disjoint) {
138 // Save dst in r11 in the disjoint case
139 __ movq(r11, dst);
140 }
141 }
142 #else
143 if (disjoint) {
144 __ mov(rdx, dst); // save 'to'
145 }
146 #endif
147 }
148
149 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
150 #ifdef _LP64
151 Register thread = r15_thread;
152 #else
153 Register thread = rax;
154 if (thread == src || thread == dst || thread == count) {
155 thread = rbx;
156 }
157 if (thread == src || thread == dst || thread == count) {
158 thread = rcx;
159 }
160 if (thread == src || thread == dst || thread == count) {
161 thread = rdx;
162 }
163 __ push(thread);
164 __ get_thread(thread);
165 #endif
166 assert_different_registers(src, dst, count, thread);
167
168 Label done;
169 // Short-circuit if count == 0.
170 __ testptr(count, count);
171 __ jcc(Assembler::zero, done);
172
173 // Avoid runtime call when not active.
174 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
175 int flags;
176 if (ShenandoahSATBBarrier && dest_uninitialized) {
177 flags = ShenandoahHeap::HAS_FORWARDED;
178 } else {
179 flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING;
180 }
181 __ testb(gc_state, flags);
182 __ jcc(Assembler::zero, done);
183
184 save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
185
186 #ifdef _LP64
187 assert(src == rdi, "expected");
188 assert(dst == rsi, "expected");
189 assert(count == rdx, "expected");
190 if (UseCompressedOops) {
191 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
192 src, dst, count);
193 } else
194 #endif
195 {
196 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
197 src, dst, count);
198 }
199
200 restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
201
202 __ bind(done);
203 NOT_LP64(__ pop(thread);)
204 }
205 }
206
207 }
208
209 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
210 Register src, Register dst, Register count) {
211 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
212 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
213 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
214 Register tmp = rax;
215
216 if (is_reference_type(type)) {
217 #ifdef _LP64
218 if (!checkcast) {
219 if (!obj_int) {
220 // Save count for barrier
221 count = r11;
222 } else if (disjoint) {
223 // Use the saved dst in the disjoint case
224 dst = r11;
225 }
226 } else {
227 tmp = rscratch1;
228 }
229 #else
230 if (disjoint) {
231 __ mov(dst, rdx); // restore 'to'
232 }
233 #endif
234 gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp);
235 }
236 }
237
238 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
239 Register obj,
240 Register pre_val,
241 Register thread,
242 Register tmp,
243 bool tosca_live,
244 bool expand_call) {
245
246 if (ShenandoahSATBBarrier) {
247 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
248 }
249 }
250
251 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
252 Register obj,
253 Register pre_val,
254 Register thread,
255 Register tmp,
256 bool tosca_live,
257 bool expand_call) {
261
262 #ifdef _LP64
263 assert(thread == r15_thread, "must be");
264 #endif // _LP64
265
266 Label done;
267 Label runtime;
268
269 assert(pre_val != noreg, "check this code");
270
271 if (obj != noreg) {
272 assert_different_registers(obj, pre_val, tmp);
273 assert(pre_val != rax, "check this code");
274 }
275
276 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
277 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
278 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
279
280 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
281 __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
282 __ jcc(Assembler::zero, done);
283
284 // Do we need to load the previous value?
285 if (obj != noreg) {
286 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
287 }
288
289 // Is the previous value null?
290 __ cmpptr(pre_val, NULL_WORD);
291 __ jcc(Assembler::equal, done);
292
293 // Can we store original value in the thread's buffer?
294 // Is index == 0?
295 // (The index field is typed as size_t.)
296
297 __ movptr(tmp, index); // tmp := *index_adr
298 __ cmpptr(tmp, 0); // tmp == 0?
299 __ jcc(Assembler::equal, runtime); // If yes, goto runtime
300
301 __ subptr(tmp, wordSize); // tmp := tmp - wordSize
627 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
628 assert_different_registers(dst, tmp1, tmp_thread);
629 if (!thread->is_valid()) {
630 thread = rdx;
631 }
632 NOT_LP64(__ get_thread(thread));
633 // Generate the SATB pre-barrier code to log the value of
634 // the referent field in an SATB buffer.
635 shenandoah_write_barrier_pre(masm /* masm */,
636 noreg /* obj */,
637 dst /* pre_val */,
638 thread /* thread */,
639 tmp1 /* tmp */,
640 true /* tosca_live */,
641 true /* expand_call */);
642
643 restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
644 }
645 }
646
647 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
648 if (!ShenandoahHeap::heap()->mode()->is_generational()) {
649 return;
650 }
651
652 // Does a store check for the oop in register obj. The content of
653 // register obj is destroyed afterwards.
654
655 ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
656 CardTable* ct = ctbs->card_table();
657
658 __ shrptr(obj, CardTable::card_shift());
659
660 Address card_addr;
661
662 // The calculation for byte_map_base is as follows:
663 // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
664 // So this essentially converts an address to a displacement and it will
665 // never need to be relocated. On 64bit however the value may be too
666 // large for a 32bit displacement.
667 intptr_t byte_map_base = (intptr_t)ct->byte_map_base();
668 if (__ is_simm32(byte_map_base)) {
669 card_addr = Address(noreg, obj, Address::times_1, byte_map_base);
670 } else {
671 // By doing it as an ExternalAddress 'byte_map_base' could be converted to a rip-relative
672 // displacement and done in a single instruction given favorable mapping and a
673 // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
674 // entry and that entry is not properly handled by the relocation code.
675 AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
676 Address index(noreg, obj, Address::times_1);
677 card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
678 }
679
680 int dirty = CardTable::dirty_card_val();
681 if (UseCondCardMark) {
682 Label L_already_dirty;
683 __ cmpb(card_addr, dirty);
684 __ jcc(Assembler::equal, L_already_dirty);
685 __ movb(card_addr, dirty);
686 __ bind(L_already_dirty);
687 } else {
688 __ movb(card_addr, dirty);
689 }
690 }
691
692 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
693 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
694
695 bool on_oop = is_reference_type(type);
696 bool in_heap = (decorators & IN_HEAP) != 0;
697 bool as_normal = (decorators & AS_NORMAL) != 0;
698 if (on_oop && in_heap) {
699 bool needs_pre_barrier = as_normal;
700
701 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
702 // flatten object address if needed
703 // We do it regardless of precise because we need the registers
704 if (dst.index() == noreg && dst.disp() == 0) {
705 if (dst.base() != tmp1) {
706 __ movptr(tmp1, dst.base());
707 }
708 } else {
709 __ lea(tmp1, dst);
710 }
711
713
714 #ifndef _LP64
715 __ get_thread(rthread);
716 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
717 imasm->save_bcp();
718 #endif
719
720 if (needs_pre_barrier) {
721 shenandoah_write_barrier_pre(masm /*masm*/,
722 tmp1 /* obj */,
723 tmp2 /* pre_val */,
724 rthread /* thread */,
725 tmp3 /* tmp */,
726 val != noreg /* tosca_live */,
727 false /* expand_call */);
728 }
729 if (val == noreg) {
730 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
731 } else {
732 iu_barrier(masm, val, tmp3);
733 // XXX: store_check missing from upstream
734 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
735 store_check(masm, tmp1);
736 }
737 NOT_LP64(imasm->restore_bcp());
738 } else {
739 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
740 }
741 }
742
743 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
744 Register obj, Register tmp, Label& slowpath) {
745 Label done;
746 // Resolve jobject
747 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
748
749 // Check for null.
750 __ testptr(obj, obj);
751 __ jcc(Assembler::zero, done);
752
753 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
754 __ testb(gc_state, ShenandoahHeap::EVACUATION);
755 __ jccb(Assembler::notZero, slowpath);
911 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
912 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
913
914 if (exchange) {
915 __ bind(L_failure);
916 __ bind(L_success);
917 } else {
918 assert(res != noreg, "need result register");
919
920 Label exit;
921 __ bind(L_failure);
922 __ xorptr(res, res);
923 __ jmpb(exit);
924
925 __ bind(L_success);
926 __ movptr(res, 1);
927 __ bind(exit);
928 }
929 }
930
931 #ifdef PRODUCT
932 #define BLOCK_COMMENT(str) /* nothing */
933 #else
934 #define BLOCK_COMMENT(str) __ block_comment(str)
935 #endif
936
937 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
938
939 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
940
941 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp) {
942 if (!ShenandoahHeap::heap()->mode()->is_generational()) {
943 return;
944 }
945
946 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
947 CardTable* ct = bs->card_table();
948 intptr_t disp = (intptr_t) ct->byte_map_base();
949
950 Label L_loop, L_done;
951 const Register end = count;
952 assert_different_registers(addr, end);
953
954 __ testl(count, count);
955 __ jcc(Assembler::zero, L_done); // zero count - nothing to do
956
957
958 #ifdef _LP64
959 __ leaq(end, Address(addr, count, TIMES_OOP, 0)); // end == addr+count*oop_size
960 __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
961 __ shrptr(addr, CardTable::card_shift());
962 __ shrptr(end, CardTable::card_shift());
963 __ subptr(end, addr); // end --> cards count
964
965 __ mov64(tmp, disp);
966 __ addptr(addr, tmp);
967 __ BIND(L_loop);
968 __ movb(Address(addr, count, Address::times_1), 0);
969 __ decrement(count);
970 __ jcc(Assembler::greaterEqual, L_loop);
971 #else
972 __ lea(end, Address(addr, count, Address::times_ptr, -wordSize));
973 __ shrptr(addr, CardTable::card_shift());
974 __ shrptr(end, CardTable::card_shift());
975 __ subptr(end, addr); // end --> count
976 __ BIND(L_loop);
977 Address cardtable(addr, count, Address::times_1, disp);
978 __ movb(cardtable, 0);
979 __ decrement(count);
980 __ jcc(Assembler::greaterEqual, L_loop);
981 #endif
982
983 __ BIND(L_done);
984 }
985
986 #undef __
987
988 #ifdef COMPILER1
989
990 #define __ ce->masm()->
991
992 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
993 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
994 // At this point we know that marking is in progress.
995 // If do_load() is true then we have to emit the
996 // load of the previous value; otherwise it has already
997 // been loaded into _pre_val.
998
999 __ bind(*stub->entry());
1000 assert(stub->pre_val()->is_register(), "Precondition.");
1001
1002 Register pre_val_reg = stub->pre_val()->as_register();
1003
1004 if (stub->do_load()) {
1005 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
1081 __ prologue("shenandoah_pre_barrier", false);
1082 // arg0 : previous value of memory
1083
1084 __ push(rax);
1085 __ push(rdx);
1086
1087 const Register pre_val = rax;
1088 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
1089 const Register tmp = rdx;
1090
1091 NOT_LP64(__ get_thread(thread);)
1092
1093 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1094 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1095
1096 Label done;
1097 Label runtime;
1098
1099 // Is SATB still active?
1100 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1101 __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
1102 __ jcc(Assembler::zero, done);
1103
1104 // Can we store original value in the thread's buffer?
1105
1106 __ movptr(tmp, queue_index);
1107 __ testptr(tmp, tmp);
1108 __ jcc(Assembler::zero, runtime);
1109 __ subptr(tmp, wordSize);
1110 __ movptr(queue_index, tmp);
1111 __ addptr(tmp, buffer);
1112
1113 // prev_val (rax)
1114 __ load_parameter(0, pre_val);
1115 __ movptr(Address(tmp, 0), pre_val);
1116 __ jmp(done);
1117
1118 __ bind(runtime);
1119
1120 __ save_live_registers_no_oop_map(true);
1121
|