1 /*
2 * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahForwarding.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "runtime/javaThread.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "utilities/macros.hpp"
38 #ifdef COMPILER1
39 #include "c1/c1_LIRAssembler.hpp"
40 #include "c1/c1_MacroAssembler.hpp"
41 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
42 #endif
43
44 #define __ masm->
45
46 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
47 if (handle_gpr) {
48 __ push_IU_state();
49 }
50
51 if (handle_fp) {
52 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
53 LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));
103 __ movflt(xmm5, Address(rsp, xmm_size * 5));
104 __ movflt(xmm6, Address(rsp, xmm_size * 6));
105 __ movflt(xmm7, Address(rsp, xmm_size * 7));
106 __ addptr(rsp, xmm_size * 8);
107 } else {
108 __ pop_FPU_state();
109 }
110 }
111
112 if (handle_gpr) {
113 __ pop_IU_state();
114 }
115 }
116
117 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
118 Register src, Register dst, Register count) {
119
120 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
121
122 if (is_reference_type(type)) {
123
124 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
125 #ifdef _LP64
126 Register thread = r15_thread;
127 #else
128 Register thread = rax;
129 if (thread == src || thread == dst || thread == count) {
130 thread = rbx;
131 }
132 if (thread == src || thread == dst || thread == count) {
133 thread = rcx;
134 }
135 if (thread == src || thread == dst || thread == count) {
136 thread = rdx;
137 }
138 __ push(thread);
139 __ get_thread(thread);
140 #endif
141 assert_different_registers(src, dst, count, thread);
142
143 Label done;
144 // Short-circuit if count == 0.
145 __ testptr(count, count);
146 __ jcc(Assembler::zero, done);
147
148 // Avoid runtime call when not active.
149 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
150 int flags;
151 if (ShenandoahSATBBarrier && dest_uninitialized) {
152 flags = ShenandoahHeap::HAS_FORWARDED;
153 } else {
154 flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
155 }
156 __ testb(gc_state, flags);
157 __ jcc(Assembler::zero, done);
158
159 save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
160
161 #ifdef _LP64
162 assert(src == rdi, "expected");
163 assert(dst == rsi, "expected");
164 assert(count == rdx, "expected");
165 if (UseCompressedOops) {
166 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
167 src, dst, count);
168 } else
169 #endif
170 {
171 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
172 src, dst, count);
173 }
174
175 restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
176
177 __ bind(done);
178 NOT_LP64(__ pop(thread);)
179 }
180 }
181
182 }
183
184 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
185 Register obj,
186 Register pre_val,
187 Register thread,
188 Register tmp,
189 bool tosca_live,
190 bool expand_call) {
191
192 if (ShenandoahSATBBarrier) {
193 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
194 }
195 }
196
197 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
198 Register obj,
199 Register pre_val,
200 Register thread,
201 Register tmp,
202 bool tosca_live,
203 bool expand_call) {
572 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
573 assert_different_registers(dst, tmp1, tmp_thread);
574 if (!thread->is_valid()) {
575 thread = rdx;
576 }
577 NOT_LP64(__ get_thread(thread));
578 // Generate the SATB pre-barrier code to log the value of
579 // the referent field in an SATB buffer.
580 shenandoah_write_barrier_pre(masm /* masm */,
581 noreg /* obj */,
582 dst /* pre_val */,
583 thread /* thread */,
584 tmp1 /* tmp */,
585 true /* tosca_live */,
586 true /* expand_call */);
587
588 restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
589 }
590 }
591
592 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
593 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
594
595 bool on_oop = is_reference_type(type);
596 bool in_heap = (decorators & IN_HEAP) != 0;
597 bool as_normal = (decorators & AS_NORMAL) != 0;
598 if (on_oop && in_heap) {
599 bool needs_pre_barrier = as_normal;
600
601 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
602 // flatten object address if needed
603 // We do it regardless of precise because we need the registers
604 if (dst.index() == noreg && dst.disp() == 0) {
605 if (dst.base() != tmp1) {
606 __ movptr(tmp1, dst.base());
607 }
608 } else {
609 __ lea(tmp1, dst);
610 }
611
614 #ifndef _LP64
615 __ get_thread(rthread);
616 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
617 imasm->save_bcp();
618 #endif
619
620 if (needs_pre_barrier) {
621 shenandoah_write_barrier_pre(masm /*masm*/,
622 tmp1 /* obj */,
623 tmp2 /* pre_val */,
624 rthread /* thread */,
625 tmp3 /* tmp */,
626 val != noreg /* tosca_live */,
627 false /* expand_call */);
628 }
629 if (val == noreg) {
630 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
631 } else {
632 iu_barrier(masm, val, tmp3);
633 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
634 }
635 NOT_LP64(imasm->restore_bcp());
636 } else {
637 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
638 }
639 }
640
641 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
642 Register obj, Register tmp, Label& slowpath) {
643 Label done;
644 // Resolve jobject
645 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
646
647 // Check for null.
648 __ testptr(obj, obj);
649 __ jcc(Assembler::zero, done);
650
651 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
652 __ testb(gc_state, ShenandoahHeap::EVACUATION);
653 __ jccb(Assembler::notZero, slowpath);
809 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
810 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
811
812 if (exchange) {
813 __ bind(L_failure);
814 __ bind(L_success);
815 } else {
816 assert(res != noreg, "need result register");
817
818 Label exit;
819 __ bind(L_failure);
820 __ xorptr(res, res);
821 __ jmpb(exit);
822
823 __ bind(L_success);
824 __ movptr(res, 1);
825 __ bind(exit);
826 }
827 }
828
829 #undef __
830
831 #ifdef COMPILER1
832
833 #define __ ce->masm()->
834
835 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
836 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
837 // At this point we know that marking is in progress.
838 // If do_load() is true then we have to emit the
839 // load of the previous value; otherwise it has already
840 // been loaded into _pre_val.
841
842 __ bind(*stub->entry());
843 assert(stub->pre_val()->is_register(), "Precondition.");
844
845 Register pre_val_reg = stub->pre_val()->as_register();
846
847 if (stub->do_load()) {
848 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
|
1 /*
2 * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
29 #include "gc/shenandoah/shenandoahForwarding.hpp"
30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
32 #include "gc/shenandoah/shenandoahRuntime.hpp"
33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
34 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
35 #include "gc/shenandoah/mode/shenandoahMode.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
49 if (handle_gpr) {
50 __ push_IU_state();
51 }
52
53 if (handle_fp) {
54 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
55 LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));
105 __ movflt(xmm5, Address(rsp, xmm_size * 5));
106 __ movflt(xmm6, Address(rsp, xmm_size * 6));
107 __ movflt(xmm7, Address(rsp, xmm_size * 7));
108 __ addptr(rsp, xmm_size * 8);
109 } else {
110 __ pop_FPU_state();
111 }
112 }
113
114 if (handle_gpr) {
115 __ pop_IU_state();
116 }
117 }
118
119 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
120 Register src, Register dst, Register count) {
121
122 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
123
124 if (is_reference_type(type)) {
125 if (ShenandoahCardBarrier) {
126 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
127 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
128 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
129
130 // We need to save the original element count because the array copy stub
131 // will destroy the value and we need it for the card marking barrier.
132 #ifdef _LP64
133 if (!checkcast) {
134 if (!obj_int) {
135 // Save count for barrier
136 __ movptr(r11, count);
137 } else if (disjoint) {
138 // Save dst in r11 in the disjoint case
139 __ movq(r11, dst);
140 }
141 }
142 #else
143 if (disjoint) {
144 __ mov(rdx, dst); // save 'to'
145 }
146 #endif
147 }
148
149 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
150 #ifdef _LP64
151 Register thread = r15_thread;
152 #else
153 Register thread = rax;
154 if (thread == src || thread == dst || thread == count) {
155 thread = rbx;
156 }
157 if (thread == src || thread == dst || thread == count) {
158 thread = rcx;
159 }
160 if (thread == src || thread == dst || thread == count) {
161 thread = rdx;
162 }
163 __ push(thread);
164 __ get_thread(thread);
165 #endif
166 assert_different_registers(src, dst, count, thread);
167
168 Label L_done;
169 // Short-circuit if count == 0.
170 __ testptr(count, count);
171 __ jcc(Assembler::zero, L_done);
172
173 // Avoid runtime call when not active.
174 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
175 int flags;
176 if (ShenandoahSATBBarrier && dest_uninitialized) {
177 flags = ShenandoahHeap::HAS_FORWARDED;
178 } else {
179 flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
180 }
181 __ testb(gc_state, flags);
182 __ jcc(Assembler::zero, L_done);
183
184 save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
185
186 #ifdef _LP64
187 assert(src == rdi, "expected");
188 assert(dst == rsi, "expected");
189 assert(count == rdx, "expected");
190 if (UseCompressedOops) {
191 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
192 src, dst, count);
193 } else
194 #endif
195 {
196 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
197 src, dst, count);
198 }
199
200 restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
201
202 __ bind(L_done);
203 NOT_LP64(__ pop(thread);)
204 }
205 }
206
207 }
208
209 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
210 Register src, Register dst, Register count) {
211
212 if (ShenandoahCardBarrier && is_reference_type(type)) {
213 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
214 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
215 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
216 Register tmp = rax;
217
218 #ifdef _LP64
219 if (!checkcast) {
220 if (!obj_int) {
221 // Save count for barrier
222 count = r11;
223 } else if (disjoint) {
224 // Use the saved dst in the disjoint case
225 dst = r11;
226 }
227 } else {
228 tmp = rscratch1;
229 }
230 #else
231 if (disjoint) {
232 __ mov(dst, rdx); // restore 'to'
233 }
234 #endif
235 gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp);
236 }
237 }
238
239 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
240 Register obj,
241 Register pre_val,
242 Register thread,
243 Register tmp,
244 bool tosca_live,
245 bool expand_call) {
246
247 if (ShenandoahSATBBarrier) {
248 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
249 }
250 }
251
252 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
253 Register obj,
254 Register pre_val,
255 Register thread,
256 Register tmp,
257 bool tosca_live,
258 bool expand_call) {
627 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
628 assert_different_registers(dst, tmp1, tmp_thread);
629 if (!thread->is_valid()) {
630 thread = rdx;
631 }
632 NOT_LP64(__ get_thread(thread));
633 // Generate the SATB pre-barrier code to log the value of
634 // the referent field in an SATB buffer.
635 shenandoah_write_barrier_pre(masm /* masm */,
636 noreg /* obj */,
637 dst /* pre_val */,
638 thread /* thread */,
639 tmp1 /* tmp */,
640 true /* tosca_live */,
641 true /* expand_call */);
642
643 restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
644 }
645 }
646
647 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
648 assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
649
650 // Does a store check for the oop in register obj. The content of
651 // register obj is destroyed afterwards.
652
653 ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
654 CardTable* ct = ctbs->card_table();
655
656 __ shrptr(obj, CardTable::card_shift());
657
658 Address card_addr;
659
660 // The calculation for byte_map_base is as follows:
661 // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
662 // So this essentially converts an address to a displacement and it will
663 // never need to be relocated. On 64-bit however the value may be too
664 // large for a 32-bit displacement.
665 intptr_t byte_map_base = (intptr_t)ct->byte_map_base();
666 if (__ is_simm32(byte_map_base)) {
667 card_addr = Address(noreg, obj, Address::times_1, byte_map_base);
668 } else {
669 // By doing it as an ExternalAddress 'byte_map_base' could be converted to a rip-relative
670 // displacement and done in a single instruction given favorable mapping and a
671 // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
672 // entry and that entry is not properly handled by the relocation code.
673 AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
674 Address index(noreg, obj, Address::times_1);
675 card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
676 }
677
678 int dirty = CardTable::dirty_card_val();
679 if (UseCondCardMark) {
680 Label L_already_dirty;
681 __ cmpb(card_addr, dirty);
682 __ jccb(Assembler::equal, L_already_dirty);
683 __ movb(card_addr, dirty);
684 __ bind(L_already_dirty);
685 } else {
686 __ movb(card_addr, dirty);
687 }
688 }
689
690 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
691 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
692
693 bool on_oop = is_reference_type(type);
694 bool in_heap = (decorators & IN_HEAP) != 0;
695 bool as_normal = (decorators & AS_NORMAL) != 0;
696 if (on_oop && in_heap) {
697 bool needs_pre_barrier = as_normal;
698
699 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
700 // flatten object address if needed
701 // We do it regardless of precise because we need the registers
702 if (dst.index() == noreg && dst.disp() == 0) {
703 if (dst.base() != tmp1) {
704 __ movptr(tmp1, dst.base());
705 }
706 } else {
707 __ lea(tmp1, dst);
708 }
709
712 #ifndef _LP64
713 __ get_thread(rthread);
714 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
715 imasm->save_bcp();
716 #endif
717
718 if (needs_pre_barrier) {
719 shenandoah_write_barrier_pre(masm /*masm*/,
720 tmp1 /* obj */,
721 tmp2 /* pre_val */,
722 rthread /* thread */,
723 tmp3 /* tmp */,
724 val != noreg /* tosca_live */,
725 false /* expand_call */);
726 }
727 if (val == noreg) {
728 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
729 } else {
730 iu_barrier(masm, val, tmp3);
731 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
732 if (ShenandoahCardBarrier) {
733 store_check(masm, tmp1);
734 }
735 }
736 NOT_LP64(imasm->restore_bcp());
737 } else {
738 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
739 }
740 }
741
742 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
743 Register obj, Register tmp, Label& slowpath) {
744 Label done;
745 // Resolve jobject
746 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
747
748 // Check for null.
749 __ testptr(obj, obj);
750 __ jcc(Assembler::zero, done);
751
752 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
753 __ testb(gc_state, ShenandoahHeap::EVACUATION);
754 __ jccb(Assembler::notZero, slowpath);
910 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
911 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
912
913 if (exchange) {
914 __ bind(L_failure);
915 __ bind(L_success);
916 } else {
917 assert(res != noreg, "need result register");
918
919 Label exit;
920 __ bind(L_failure);
921 __ xorptr(res, res);
922 __ jmpb(exit);
923
924 __ bind(L_success);
925 __ movptr(res, 1);
926 __ bind(exit);
927 }
928 }
929
930 #ifdef PRODUCT
931 #define BLOCK_COMMENT(str) /* nothing */
932 #else
933 #define BLOCK_COMMENT(str) __ block_comment(str)
934 #endif
935
936 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
937
938 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
939
940 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
941 Register addr, Register count,
942 Register tmp) {
943 assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
944
945 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
946 CardTable* ct = bs->card_table();
947 intptr_t disp = (intptr_t) ct->byte_map_base();
948
949 Label L_loop, L_done;
950 const Register end = count;
951 assert_different_registers(addr, end);
952
953 // Zero count? Nothing to do.
954 __ testl(count, count);
955 __ jccb(Assembler::zero, L_done);
956
957 #ifdef _LP64
958 __ leaq(end, Address(addr, count, TIMES_OOP, 0)); // end == addr+count*oop_size
959 __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
960 __ shrptr(addr, CardTable::card_shift());
961 __ shrptr(end, CardTable::card_shift());
962 __ subptr(end, addr); // end --> cards count
963
964 __ mov64(tmp, disp);
965 __ addptr(addr, tmp);
966
967 __ BIND(L_loop);
968 __ movb(Address(addr, count, Address::times_1), 0);
969 __ decrement(count);
970 __ jccb(Assembler::greaterEqual, L_loop);
971 #else
972 __ lea(end, Address(addr, count, Address::times_ptr, -wordSize));
973 __ shrptr(addr, CardTable::card_shift());
974 __ shrptr(end, CardTable::card_shift());
975 __ subptr(end, addr); // end --> count
976
977 __ BIND(L_loop);
978 Address cardtable(addr, count, Address::times_1, disp);
979 __ movb(cardtable, 0);
980 __ decrement(count);
981 __ jccb(Assembler::greaterEqual, L_loop);
982 #endif
983
984 __ BIND(L_done);
985 }
986
987 #undef __
988
989 #ifdef COMPILER1
990
991 #define __ ce->masm()->
992
993 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
994 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
995 // At this point we know that marking is in progress.
996 // If do_load() is true then we have to emit the
997 // load of the previous value; otherwise it has already
998 // been loaded into _pre_val.
999
1000 __ bind(*stub->entry());
1001 assert(stub->pre_val()->is_register(), "Precondition.");
1002
1003 Register pre_val_reg = stub->pre_val()->as_register();
1004
1005 if (stub->do_load()) {
1006 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
|