5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 source_hpp %{
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 %}
29
30 encode %{
31 enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
32 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
33 Register tmp = $tmp$$Register;
34 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
35 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
36 /*acquire*/ false, /*release*/ true, /*is_cae*/ false, $res$$Register);
37 %}
38
39 enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
40 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
41 Register tmp = $tmp$$Register;
42 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
43 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
44 /*acquire*/ true, /*release*/ true, /*is_cae*/ false, $res$$Register);
45 %}
46 %}
47
48 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
49
50 match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
51 ins_cost(2 * VOLATILE_REF_COST);
52
53 effect(TEMP tmp, KILL cr);
54
55 format %{
56 "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
57 %}
58
59 ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp, res));
60
61 ins_pipe(pipe_slow);
62 %}
63
64 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
65
66 match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
67 ins_cost(2 * VOLATILE_REF_COST);
68
69 effect(TEMP tmp, KILL cr);
70
71 format %{
72 "cmpxchgw_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
73 %}
74
75 ins_encode %{
76 Register tmp = $tmp$$Register;
77 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
78 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*is_cae*/ false, $res$$Register);
79 %}
80
81 ins_pipe(pipe_slow);
82 %}
83
84 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
85
86 predicate(needs_acquiring_load_exclusive(n));
87 match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
88 ins_cost(VOLATILE_REF_COST);
89
90 effect(TEMP tmp, KILL cr);
91
92 format %{
93 "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
94 %}
95
96 ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp, res));
97
98 ins_pipe(pipe_slow);
99 %}
100
101 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
102
103 predicate(needs_acquiring_load_exclusive(n));
104 match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
105 ins_cost(VOLATILE_REF_COST);
106
107 effect(TEMP tmp, KILL cr);
108
109 format %{
110 "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
111 %}
112
113 ins_encode %{
114 Register tmp = $tmp$$Register;
115 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
116 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*is_cae*/ false, $res$$Register);
117 %}
118
119 ins_pipe(pipe_slow);
120 %}
121
122 instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
123 match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval)));
124 ins_cost(2 * VOLATILE_REF_COST);
125 effect(TEMP_DEF res, TEMP tmp, KILL cr);
126 format %{
127 "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
128 %}
129 ins_encode %{
130 Register tmp = $tmp$$Register;
131 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
132 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
133 /*acquire*/ false, /*release*/ true, /*is_cae*/ true, $res$$Register);
134 %}
135 ins_pipe(pipe_slow);
136 %}
137
138 instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
139 match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval)));
140 ins_cost(2 * VOLATILE_REF_COST);
141 effect(TEMP_DEF res, TEMP tmp, KILL cr);
142 format %{
143 "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
144 %}
145 ins_encode %{
146 Register tmp = $tmp$$Register;
147 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
148 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
149 /*acquire*/ false, /*release*/ true, /*is_cae*/ true, $res$$Register);
150 %}
151 ins_pipe(pipe_slow);
152 %}
153
154 instruct compareAndExchangeNAcq_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
155 predicate(needs_acquiring_load_exclusive(n));
156 match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval)));
157 ins_cost(VOLATILE_REF_COST);
158 effect(TEMP_DEF res, TEMP tmp, KILL cr);
159 format %{
160 "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
161 %}
162 ins_encode %{
163 Register tmp = $tmp$$Register;
164 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
165 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
166 /*acquire*/ true, /*release*/ true, /*is_cae*/ true, $res$$Register);
167 %}
168 ins_pipe(pipe_slow);
169 %}
170
171 instruct compareAndExchangePAcq_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
172 predicate(needs_acquiring_load_exclusive(n));
173 match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval)));
174 ins_cost(VOLATILE_REF_COST);
175 effect(TEMP_DEF res, TEMP tmp, KILL cr);
176 format %{
177 "cmpxchg_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
178 %}
179 ins_encode %{
180 Register tmp = $tmp$$Register;
181 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
182 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
183 /*acquire*/ true, /*release*/ true, /*is_cae*/ true, $res$$Register);
184 %}
185 ins_pipe(pipe_slow);
186 %}
187
188 instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
189 match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval)));
190 ins_cost(2 * VOLATILE_REF_COST);
191 effect(TEMP tmp, KILL cr);
192 format %{
193 "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
194 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
195 %}
196 ins_encode %{
197 Register tmp = $tmp$$Register;
198 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
199 // Weak is not currently supported by ShenandoahBarrierSet::cmpxchg_oop
200 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
201 /*acquire*/ false, /*release*/ true, /*is_cae*/ false, $res$$Register);
202 %}
203 ins_pipe(pipe_slow);
204 %}
205
206 instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
207 match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval)));
208 ins_cost(2 * VOLATILE_REF_COST);
209 effect(TEMP tmp, KILL cr);
210 format %{
211 "cmpxchg_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
212 %}
213 ins_encode %{
214 Register tmp = $tmp$$Register;
215 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
216 // Weak is not currently supported by ShenandoahBarrierSet::cmpxchg_oop
217 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
218 /*acquire*/ false, /*release*/ true, /*is_cae*/ false, $res$$Register);
219 %}
220 ins_pipe(pipe_slow);
221 %}
222
223 instruct weakCompareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
224 predicate(needs_acquiring_load_exclusive(n));
225 match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval)));
226 ins_cost(VOLATILE_REF_COST);
227 effect(TEMP tmp, KILL cr);
228 format %{
229 "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
230 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
231 %}
232 ins_encode %{
233 Register tmp = $tmp$$Register;
234 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
235 // Weak is not currently supported by ShenandoahBarrierSet::cmpxchg_oop
236 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
237 /*acquire*/ true, /*release*/ true, /*is_cae*/ false, $res$$Register);
238 %}
239 ins_pipe(pipe_slow);
240 %}
241
242 instruct weakCompareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
243 predicate(needs_acquiring_load_exclusive(n));
244 match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval)));
245 ins_cost(VOLATILE_REF_COST);
246 effect(TEMP tmp, KILL cr);
247 format %{
248 "cmpxchg_acq_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
249 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
250 %}
251 ins_encode %{
252 Register tmp = $tmp$$Register;
253 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
254 // Weak is not currently supported by ShenandoahBarrierSet::cmpxchg_oop
255 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
256 /*acquire*/ true, /*release*/ true, /*is_cae*/ false, $res$$Register);
257 %}
258 ins_pipe(pipe_slow);
259 %}
|
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 source %{
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 %}
30
31 instruct storeP_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
32 %{
33 match(Set mem (StoreP mem src));
34 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
35 effect(TEMP tmp, KILL cr);
36 ins_cost(INSN_COST);
37 format %{ "str $src, $mem\t# ptr" %}
38 ins_encode %{
39 // Loads gc_state
40 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
41 __ ldrb($tmp$$Register, gcs_addr);
42
43 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
44 $mem$$Register /* addr, used in slow path only */,
45 noreg /* */,
46 $tmp$$Register /* gc_state on fast path */,
47 false /* encoded_preval */);
48 __ str($src$$Register, $mem$$Register);
49
50 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
51 $mem$$Register /* addr */,
52 $tmp$$Register /* tmp */);
53 %}
54 ins_pipe(istore_reg_mem);
55 %}
56
57 instruct storePVolatile_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
58 %{
59 match(Set mem (StoreP mem src));
60 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
61 effect(TEMP tmp, KILL cr);
62 ins_cost(VOLATILE_REF_COST);
63 format %{ "stlr $src, $mem\t# ptr" %}
64 ins_encode %{
65 // Loads gc_state
66 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
67 __ ldrb($tmp$$Register, gcs_addr);
68
69 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
70 $mem$$Register /* addr, used in slow path only */,
71 noreg /* */,
72 $tmp$$Register /* gc_state on fas path */,
73 false /* encoded_preval */);
74
75 __ stlr($src$$Register, $mem$$Register);
76
77 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
78 $mem$$Register /* addr */,
79 $tmp$$Register /* tmp */);
80 %}
81 ins_pipe(pipe_class_memory);
82 %}
83
84 instruct storeN_shenandoah(indirect mem, iRegN src, iRegPNoSp tmp, rFlagsReg cr)
85 %{
86 match(Set mem (StoreN mem src));
87 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
88 effect(TEMP tmp, KILL cr);
89 ins_cost(INSN_COST);
90 format %{ "strw $src, $mem\t# compressed ptr" %}
91 ins_encode %{
92 // Loads gc_state
93 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
94 __ ldrb($tmp$$Register, gcs_addr);
95
96 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
97 $mem$$Register /* addr, used in slow path only */,
98 noreg /* pre_val, used in slow path only */,
99 $tmp$$Register /* gc_state on fas path */,
100 false /* encoded_preval */);
101
102 __ strw($src$$Register, $mem$$Register);
103
104 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
105 $mem$$Register /* addr */,
106 $tmp$$Register /* tmp */);
107 %}
108 ins_pipe(istore_reg_mem);
109 %}
110
111 instruct storeNVolatile_shenandoah(indirect mem, iRegN src, iRegPNoSp tmp, rFlagsReg cr)
112 %{
113 match(Set mem (StoreN mem src));
114 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
115 effect(TEMP tmp, KILL cr);
116 ins_cost(VOLATILE_REF_COST);
117 format %{ "stlrw $src, $mem\t# compressed ptr" %}
118 ins_encode %{
119 // Loads gc_state
120 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
121 __ ldrb($tmp$$Register, gcs_addr);
122
123 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
124 $mem$$Register /* addr, used in slow path only */,
125 noreg /* pre_val, used in slow path only */,
126 $tmp$$Register /* gc_state on fas path */,
127 false /* encoded_preval */);
128
129 __ stlrw($src$$Register, $mem$$Register);
130
131 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
132 $mem$$Register /* addr */,
133 $tmp$$Register /* tmp */);
134 %}
135 ins_pipe(pipe_class_memory);
136 %}
137
138 instruct encodePAndStoreN_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
139 %{
140 match(Set mem (StoreN mem (EncodeP src)));
141 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
142 effect(TEMP tmp, KILL cr);
143 ins_cost(INSN_COST);
144 format %{ "encode_heap_oop $tmp, $src\n\t"
145 "strw $tmp, $mem\t# compressed ptr" %}
146 ins_encode %{
147 // Loads gc_state
148 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
149 __ ldrb($tmp$$Register, gcs_addr);
150
151 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
152 $mem$$Register /* addr, used in slow path only */,
153 noreg /* pre_val, used in slow path only */,
154 $tmp$$Register /* gc_state on fas path */,
155 false /* encoded_preval */);
156
157 if ((barrier_data() & ShenandoahBarrierCardMarkNotNull) == 0) {
158 __ encode_heap_oop($tmp$$Register, $src$$Register);
159 } else {
160 __ encode_heap_oop_not_null($tmp$$Register, $src$$Register);
161 }
162
163 __ strw($tmp$$Register, $mem$$Register);
164
165 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
166 $mem$$Register /* addr */,
167 $tmp$$Register /* tmp */);
168 %}
169 ins_pipe(istore_reg_mem);
170 %}
171
172 instruct encodePAndStoreNVolatile_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
173 %{
174 match(Set mem (StoreN mem (EncodeP src)));
175 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
176 effect(TEMP tmp, KILL cr);
177 ins_cost(VOLATILE_REF_COST);
178 format %{ "encode_heap_oop $tmp, $src\n\t"
179 "stlrw $tmp, $mem\t# compressed ptr" %}
180 ins_encode %{
181 // Loads gc_state
182 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
183 __ ldrb($tmp$$Register, gcs_addr);
184
185 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
186 $mem$$Register /* addr, used in slow path only */,
187 noreg /* pre_val, used in slow path only */,
188 $tmp$$Register /* gc_state on fas path */,
189 false /* encoded_preval */);
190
191 if ((barrier_data() & ShenandoahBarrierCardMarkNotNull) == 0) {
192 __ encode_heap_oop($tmp$$Register, $src$$Register);
193 } else {
194 __ encode_heap_oop_not_null($tmp$$Register, $src$$Register);
195 }
196
197 __ stlrw($tmp$$Register, $mem$$Register);
198
199 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
200 $mem$$Register /* addr */,
201 $tmp$$Register /* tmp */);
202 %}
203 ins_pipe(pipe_class_memory);
204 %}
205
206 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegPNoSp oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
207 %{
208 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
209 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
210 ins_cost(2 * VOLATILE_REF_COST);
211 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
212 format %{
213 "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
214 %}
215 ins_encode %{
216 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
217 guarantee(!UseCompressedOops, "must not be compressed oops");
218
219 // Loads gc_state
220 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
221 __ ldrb($tmp1$$Register, gcs_addr);
222
223 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
224 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
225 noreg /* addr */,
226 $oldval$$Register /* pre_val, because addr above is noreg this will be readonly always */,
227 $tmp1$$Register /* gc_state */,
228 true);
229 }
230
231 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
232 $mem$$base$$Register,
233 $oldval$$Register,
234 $newval$$Register,
235 $res$$Register,
236 $tmp1$$Register, /* gc_state */
237 $tmp2$$Register,
238 /*acquire*/ false,
239 /*release*/ true,
240 /*weak*/ false,
241 /*is_cae*/ false);
242
243 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
244 $mem$$Register /* addr */,
245 $tmp1$$Register /* tmp */);
246 %}
247
248 ins_pipe(pipe_slow);
249 %}
250
251 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
252 %{
253 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
254 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
255 ins_cost(2 * VOLATILE_REF_COST);
256 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
257 format %{
258 "cmpxchgw_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
259 %}
260 ins_encode %{
261 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
262 guarantee(UseCompressedOops, "must be compressed oops");
263
264 // Loads gc_state
265 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
266 __ ldrb($tmp1$$Register, gcs_addr);
267
268 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
269 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
270 noreg /* addr */,
271 $oldval$$Register /* fastpath: unused, slowpath: preval */,
272 $tmp1$$Register /* fastpath: gc_state, slowpath: tmp */,
273 true);
274 }
275
276 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
277 $mem$$base$$Register,
278 $oldval$$Register,
279 $newval$$Register,
280 $res$$Register,
281 $tmp1$$Register, /* gc_state */
282 $tmp2$$Register,
283 /*acquire*/ false,
284 /*release*/ true,
285 /*weak*/ false,
286 /*is_cae*/ false);
287
288 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
289 $mem$$Register /* addr */,
290 $tmp1$$Register /* tmp */);
291 %}
292
293 ins_pipe(pipe_slow);
294 %}
295
296 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegINoSp tmp1, iRegINoSp tmp2, rFlagsReg cr)
297 %{
298 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
299 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
300 ins_cost(VOLATILE_REF_COST);
301 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
302 format %{
303 "cmpxchg_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
304 %}
305 ins_encode %{
306 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
307
308 // Loads gc_state
309 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
310 __ ldrb($tmp1$$Register, gcs_addr);
311
312 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
313 noreg /* addr */,
314 $oldval$$Register /* pre_val, only used in slow path */,
315 $tmp1$$Register /* gc_state */,
316 false /* encoded_preval */);
317
318 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
319 $mem$$base$$Register,
320 $oldval$$Register,
321 $newval$$Register,
322 $res$$Register,
323 $tmp1$$Register,
324 $tmp2$$Register,
325 /*acquire*/ true,
326 /*release*/ true,
327 /*weak*/ false,
328 /*is_cae*/ false);
329
330 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
331 $mem$$Register /* addr */,
332 $tmp1$$Register /* tmp */);
333 %}
334
335 ins_pipe(pipe_slow);
336 %}
337
338 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegPNoSp oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
339 %{
340 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
341 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
342 ins_cost(VOLATILE_REF_COST);
343 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
344 format %{
345 "cmpxchgw_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
346 %}
347 ins_encode %{
348 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
349 guarantee(UseCompressedOops, "must be compressed oops");
350
351 // Loads gc_state
352 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
353 __ ldrb($tmp1$$Register, gcs_addr);
354
355 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
356 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
357 noreg /* addr */,
358 $oldval$$Register /* pre_val, only used in slow path */,
359 $tmp1$$Register /* gc_state */,
360 true /* encoded_preval */);
361 }
362
363 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
364 $mem$$base$$Register,
365 $oldval$$Register,
366 $newval$$Register,
367 $res$$Register,
368 $tmp1$$Register, /* gc_state */
369 $tmp2$$Register,
370 /*acquire*/ true,
371 /*release*/ true,
372 /*weak*/ false,
373 /*is_cae*/ false);
374
375 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
376 $mem$$Register /* addr */,
377 $tmp1$$Register /* tmp */);
378 %}
379
380 ins_pipe(pipe_slow);
381 %}
382
383 instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegPNoSp oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
384 %{
385 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
386 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0 && !needs_acquiring_load_exclusive(n));
387 ins_cost(2 * VOLATILE_REF_COST);
388 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
389 format %{
390 "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
391 %}
392 ins_encode %{
393 __ block_comment("compareAndExchangeN_shenandoah");
394
395 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
396 guarantee(UseCompressedOops, "must be compressed oops");
397
398 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
399
400 // Loads gc_state
401 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
402 __ ldrb($tmp1$$Register, gcs_addr);
403
404 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
405 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
406 noreg /* addr */,
407 $oldval$$Register /* pre_val */,
408 $tmp1$$Register /* gc_state */,
409 true /* encoded_preval */);
410 }
411
412 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
413 $mem$$base$$Register,
414 $oldval$$Register,
415 $newval$$Register,
416 $res$$Register,
417 $tmp1$$Register, /* gc_state */
418 $tmp2$$Register,
419 /*acquire*/ false,
420 /*release*/ true,
421 /*weak*/ false,
422 /*is_cae*/ true);
423
424 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
425 $res$$Register /* obj */,
426 $mem$$Register /* addr */,
427 true /* narrow */,
428 maybe_null,
429 $tmp1$$Register /* gc_state */);
430
431 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
432 $mem$$Register /* addr */,
433 $tmp1$$Register /* tmp */);
434
435 __ block_comment("} compareAndExchangeN_shenandoah");
436 %}
437 ins_pipe(pipe_slow);
438 %}
439
440 instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
441 %{
442 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
443 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
444 ins_cost(2 * VOLATILE_REF_COST);
445 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
446 format %{
447 "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
448 %}
449 ins_encode %{
450 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
451
452 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
453
454 // Loads gc_state
455 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
456 __ ldrb($tmp1$$Register, gcs_addr);
457
458 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
459 noreg /* addr */,
460 $oldval$$Register /* pre_val */,
461 $tmp1$$Register /* gc_state */,
462 false /* encoded_preval */);
463
464 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
465 $mem$$base$$Register,
466 $oldval$$Register,
467 $newval$$Register,
468 $res$$Register,
469 $tmp1$$Register, /* gc_state */
470 $tmp2$$Register,
471 /*acquire*/ false,
472 /*release*/ true,
473 /*weak*/ false,
474 /*is_cae*/ true);
475
476 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
477 $res$$Register /* obj */,
478 $mem$$Register /* addr */,
479 false /* narrow */,
480 maybe_null,
481 $tmp1$$Register /* gc_state */);
482
483 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
484 $mem$$Register /* addr */,
485 $tmp1$$Register /* tmp */);
486 %}
487 ins_pipe(pipe_slow);
488 %}
489
490 instruct compareAndExchangeNAcq_shenandoah(iRegNNoSp res, indirect mem, iRegNNoSp oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
491 %{
492 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
493 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
494 ins_cost(VOLATILE_REF_COST);
495 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
496 format %{
497 "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
498 %}
499 ins_encode %{
500 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
501 guarantee(UseCompressedOops, "must be compressed oops");
502
503 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
504
505 // Loads gc_state
506 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
507 __ ldrb($tmp1$$Register, gcs_addr);
508
509 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
510 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
511 noreg /* addr */,
512 $oldval$$Register /* pre_val */,
513 $tmp1$$Register /* gc_state */,
514 true /* encoded_preval */);
515 }
516
517 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
518 $mem$$base$$Register,
519 $oldval$$Register,
520 $newval$$Register,
521 $res$$Register,
522 $tmp1$$Register, /* gc_state */
523 $tmp2$$Register,
524 /*acquire*/ true,
525 /*release*/ true,
526 /*weak*/ false,
527 /*is_cae*/ true);
528
529 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
530 $res$$Register /* obj */,
531 $mem$$Register /* addr */,
532 true /* narrow */,
533 maybe_null,
534 $tmp1$$Register /* gc_state */);
535
536 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
537 $mem$$Register /* addr */,
538 $tmp1$$Register /* tmp */);
539 %}
540 ins_pipe(pipe_slow);
541 %}
542
543 instruct compareAndExchangePAcq_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
544 %{
545 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
546 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
547 ins_cost(VOLATILE_REF_COST);
548 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
549 format %{
550 "cmpxchg_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
551 %}
552 ins_encode %{
553 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
554
555 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
556
557 // Loads gc_state
558 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
559 __ ldrb($tmp1$$Register, gcs_addr);
560
561 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
562 noreg /* addr */,
563 $oldval$$Register /* pre_val */,
564 $tmp1$$Register /* gc_state */,
565 false /* encoded_preval */);
566
567 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
568 $mem$$base$$Register,
569 $oldval$$Register,
570 $newval$$Register,
571 $res$$Register,
572 $tmp1$$Register, /* gc_state */
573 $tmp2$$Register,
574 /*acquire*/ true,
575 /*release*/ true,
576 /*weak*/ false,
577 /*is_cae*/ true);
578
579 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
580 $res$$Register /* obj */,
581 $mem$$Register /* addr */,
582 false /* narrow */,
583 maybe_null,
584 $tmp1$$Register /* gc_state */);
585
586 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
587 $mem$$Register /* addr */,
588 $tmp1$$Register /* tmp */);
589 %}
590 ins_pipe(pipe_slow);
591 %}
592
593 instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
594 %{
595 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
596 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
597 ins_cost(2 * VOLATILE_REF_COST);
598 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
599 format %{
600 "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
601 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
602 %}
603 ins_encode %{
604 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
605 guarantee(UseCompressedOops, "must be compressed oops");
606
607 // Loads gc_state
608 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
609 __ ldrb($tmp1$$Register, gcs_addr);
610
611 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
612 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
613 noreg /* addr */,
614 $oldval$$Register /* pre_val */,
615 $tmp1$$Register /* gc_state */,
616 true /* encoded_preval */);
617 }
618
619 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
620 $mem$$base$$Register,
621 $oldval$$Register,
622 $newval$$Register,
623 $res$$Register,
624 $tmp1$$Register, /* gc_state */
625 $tmp2$$Register,
626 /*acquire*/ false,
627 /*release*/ true,
628 /*weak*/ true,
629 /*is_cae*/ false);
630
631 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
632 $mem$$Register /* addr */,
633 $tmp1$$Register /* tmp */);
634 %}
635 ins_pipe(pipe_slow);
636 %}
637
638 instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegINoSp tmp1, iRegINoSp tmp2, rFlagsReg cr)
639 %{
640 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
641 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
642 ins_cost(2 * VOLATILE_REF_COST);
643 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
644 format %{
645 "cmpxchg_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
646 %}
647 ins_encode %{
648 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
649
650 // Loads gc_state
651 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
652 __ ldrb($tmp1$$Register, gcs_addr);
653
654 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
655 noreg /* addr */,
656 $oldval$$Register /* pre_val */,
657 $tmp1$$Register /* gc_state */,
658 false /* encoded_preval */);
659
660 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
661 $mem$$base$$Register,
662 $oldval$$Register,
663 $newval$$Register,
664 $res$$Register,
665 $tmp1$$Register, /* gc_state */
666 $tmp2$$Register,
667 /*acquire*/ false,
668 /*release*/ true,
669 /*weak*/ true,
670 /*is_cae*/ false);
671
672 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
673 $mem$$Register /* addr */,
674 $tmp1$$Register /* tmp */);
675 %}
676 ins_pipe(pipe_slow);
677 %}
678
679 instruct weakCompareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
680 %{
681 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
682 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
683 ins_cost(VOLATILE_REF_COST);
684 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
685 format %{
686 "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
687 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
688 %}
689 ins_encode %{
690 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
691 guarantee(UseCompressedOops, "must be compressed oops");
692
693 // Loads gc_state
694 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
695 __ ldrb($tmp1$$Register, gcs_addr);
696
697 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
698 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
699 noreg /* addr */,
700 $oldval$$Register /* pre_val */,
701 $tmp1$$Register /* gc_state */,
702 true /* encoded_preval */);
703 }
704
705 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
706 $mem$$base$$Register,
707 $oldval$$Register,
708 $newval$$Register,
709 $res$$Register,
710 $tmp1$$Register, /* gc_state */
711 $tmp2$$Register,
712 /*acquire*/ true,
713 /*release*/ true,
714 /*weak*/ true,
715 /*is_cae*/ false);
716
717 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
718 $mem$$Register /* addr */,
719 $tmp1$$Register /* tmp */);
720 %}
721 ins_pipe(pipe_slow);
722 %}
723
724 instruct weakCompareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegPNoSp oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
725 %{
726 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
727 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
728 ins_cost(VOLATILE_REF_COST);
729 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
730 format %{
731 "cmpxchg_acq_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
732 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
733 %}
734 ins_encode %{
735 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
736
737 // Loads gc_state
738 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
739 __ ldrb($tmp1$$Register, gcs_addr);
740
741 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
742 noreg /* addr */,
743 $oldval$$Register /* pre_val */,
744 $tmp1$$Register /* gc_state */,
745 false /* encoded_preval */);
746
747 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
748 $mem$$base$$Register,
749 $oldval$$Register,
750 $newval$$Register,
751 $res$$Register,
752 $tmp1$$Register,
753 $tmp2$$Register,
754 /*acquire*/ true,
755 /*release*/ true,
756 /*weak*/ true,
757 /*is_cae*/ false);
758
759 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
760 $mem$$Register /* addr */,
761 $tmp1$$Register /* tmp */);
762 %}
763 ins_pipe(pipe_slow);
764 %}
765
766 instruct getAndSetP_shenandoah(indirect mem, iRegP newval, iRegPNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
767 %{
768 match(Set preval (GetAndSetP mem newval));
769 predicate(UseShenandoahGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
770 effect(TEMP_DEF preval, TEMP tmp, KILL cr);
771 ins_cost(2 * VOLATILE_REF_COST);
772 format %{ "atomic_xchg $preval, $newval, [$mem]" %}
773 ins_encode %{
774 assert_different_registers($mem$$Register, $newval$$Register);
775
776 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
777
778 __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
779
780 // Loads gc_state
781 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
782 __ ldrb($tmp$$Register, gcs_addr);
783
784 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
785 noreg /* addr */,
786 $preval$$Register /* pre_val */,
787 $tmp$$Register /* gc_state */,
788 false /* encoded_preval */);
789
790 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
791 $preval$$Register /* obj */,
792 $mem$$Register /* addr */,
793 false /* narrow */,
794 maybe_null,
795 $tmp$$Register /* gc_state */);
796
797 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
798 $mem$$Register /* addr */,
799 $tmp$$Register /* tmp */);
800 %}
801 ins_pipe(pipe_serial);
802 %}
803
804 instruct getAndSetPAcq_shenandoah(indirect mem, iRegP newval, iRegPNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
805 %{
806 match(Set preval (GetAndSetP mem newval));
807 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
808 effect(TEMP_DEF preval, TEMP tmp, KILL cr);
809 ins_cost(VOLATILE_REF_COST);
810 format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %}
811 ins_encode %{
812 assert_different_registers($mem$$Register, $newval$$Register);
813
814 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
815
816 __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
817
818 // Loads gc_state
819 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
820 __ ldrb($tmp$$Register, gcs_addr);
821
822 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
823 noreg /* addr */,
824 $preval$$Register /* pre_val */,
825 $tmp$$Register,
826 false /* encoded_preval */);
827
828 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
829 $preval$$Register /* obj */,
830 $mem$$Register /* addr */,
831 false /* narrow */,
832 maybe_null,
833 $tmp$$Register /* gc_state */);
834
835 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
836 $mem$$Register /* addr */,
837 $tmp$$Register /* tmp */);
838 %}
839 ins_pipe(pipe_serial);
840 %}
841
842 instruct getAndSetN_shenandoah(indirect mem, iRegN newval, iRegNNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
843 %{
844 match(Set preval (GetAndSetN mem newval));
845 predicate(UseShenandoahGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
846 effect(TEMP_DEF preval, TEMP tmp, KILL cr);
847 ins_cost(2 * VOLATILE_REF_COST);
848 format %{ "atomic_xchgw $preval, $newval, [$mem]" %}
849 ins_encode %{
850 assert_different_registers($mem$$Register, $newval$$Register);
851
852 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
853
854 __ atomic_xchgw($preval$$Register, $newval$$Register, $mem$$Register);
855
856 // Loads gc_state
857 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
858 __ ldrb($tmp$$Register, gcs_addr);
859
860 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
861 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
862 noreg /* addr */,
863 $preval$$Register /* pre_val */,
864 $tmp$$Register /* gc_state */,
865 true /* encoded_preval */);
866 }
867
868 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
869 $preval$$Register /* obj */,
870 $mem$$Register /* addr */,
871 true /* narrow */,
872 maybe_null,
873 $tmp$$Register /* gc_state */);
874
875 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
876 $mem$$Register /* addr */,
877 $tmp$$Register /* tmp */);
878 %}
879 ins_pipe(pipe_serial);
880 %}
881
882 instruct getAndSetNAcq_shenandoah(indirect mem, iRegN newval, iRegNNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
883 %{
884 match(Set preval (GetAndSetN mem newval));
885 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
886 effect(TEMP_DEF preval, TEMP tmp, KILL cr);
887 ins_cost(VOLATILE_REF_COST);
888 format %{ "atomic_xchgw_acq $preval, $newval, [$mem]" %}
889 ins_encode %{
890 assert_different_registers($mem$$Register, $newval$$Register);
891
892 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
893
894 __ atomic_xchgalw($preval$$Register, $newval$$Register, $mem$$Register);
895
896 // Loads gc_state
897 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
898 __ ldrb($tmp$$Register, gcs_addr);
899
900 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
901 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
902 noreg /* addr */,
903 $preval$$Register /* pre_val */,
904 $tmp$$Register /* gc_state */,
905 true /* encoded_preval */);
906 }
907 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
908 $preval$$Register /* obj */,
909 $mem$$Register /* addr */,
910 true /* narrow */,
911 maybe_null,
912 $tmp$$Register /* gc_state */);
913
914 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
915 $mem$$Register /* addr */,
916 $tmp$$Register /* tmp */);
917 %}
918 ins_pipe(pipe_serial);
919 %}
920
921 instruct loadP_shenandoah(iRegPNoSp dst, indirect mem, iRegPNoSp tmp, rFlagsReg cr)
922 %{
923 // This instruction does not need an acquiring counterpart because it is only
924 // used for reference loading (Reference::get()).
925 match(Set dst (LoadP mem));
926 predicate(UseShenandoahGC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
927 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
928 ins_cost(4 * INSN_COST);
929 format %{ "ldr $dst, $mem\t# ptr" %}
930 ins_encode %{
931 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
932
933 __ ldr($dst$$Register, $mem$$Register);
934
935 // Loads gc_state
936 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
937 __ ldrb($tmp$$Register, gcs_addr);
938
939 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
940 noreg /* addr */,
941 $dst$$Register /* pre_val */,
942 $tmp$$Register /* gc_state */,
943 false /* encoded_preval */);
944
945 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
946 $dst$$Register /* obj */,
947 $mem$$Register /* addr */,
948 false /* narrow */,
949 maybe_null,
950 $tmp$$Register /* gc_state */);
951 %}
952 ins_pipe(pipe_class_memory);
953 %}
954
955 instruct loadP_volatile_shenandoah(iRegPNoSp dst, indirect mem, iRegPNoSp tmp, rFlagsReg cr)
956 %{
957 // This instruction does not need an acquiring counterpart because it is only
958 // used for reference loading (Reference::get()).
959 match(Set dst (LoadP mem));
960 predicate(UseShenandoahGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
961 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
962 ins_cost(4 * INSN_COST);
963 format %{ "ldar $dst, $mem\t# ptr" %}
964 ins_encode %{
965 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
966
967 __ ldar($dst$$Register, $mem$$Register);
968
969 // Loads gc_state
970 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
971 __ ldrb($tmp$$Register, gcs_addr);
972
973 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
974 noreg /* addr */,
975 $dst$$Register /* pre_val */,
976 $tmp$$Register /* gc_state */,
977 false /* encoded_preval */);
978
979 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
980 $dst$$Register /* obj */,
981 $mem$$Register /* addr */,
982 false /* narrow */,
983 maybe_null,
984 $tmp$$Register /* gc_state */);
985 %}
986 ins_pipe(pipe_class_memory);
987 %}
988
989 instruct loadN_decodeN_shenandoah(iRegPNoSp dst, indirect mem, iRegPNoSp tmp, rFlagsReg cr)
990 %{
991 match(Set dst (DecodeN (LoadN mem)));
992 predicate(UseShenandoahGC && !needs_acquiring_load(n->in(1)) && n->in(1)->as_Load()->barrier_data() != 0);
993 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
994 ins_cost(INSN_COST * 2);
995 format %{ "ldrw $dst, $mem\t# decode(load(n))" %}
996 ins_encode %{
997 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
998
999 // Loads compressed pointer
1000 __ ldrw($dst$$Register, $mem$$Register);
1001
1002 // Loads gc_state
1003 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1004 __ ldrb($tmp$$Register, gcs_addr);
1005
1006 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
1007 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
1008 noreg /* addr */,
1009 $dst$$Register /* pre_val */,
1010 $tmp$$Register /* gc_state */,
1011 true /* encoded_preval */);
1012 }
1013
1014 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
1015 $dst$$Register /* obj */,
1016 $mem$$Register /* addr */,
1017 true /* narrow */,
1018 maybe_null,
1019 $tmp$$Register /* gc_state */);
1020
1021 __ decode_heap_oop($dst$$Register, $dst$$Register);
1022 %}
1023 ins_pipe(pipe_class_memory);
1024 %}
1025
1026 instruct loadN_shenandoah(iRegNNoSp dst, indirect mem, iRegNNoSp tmp, rFlagsReg cr)
1027 %{
1028 match(Set dst (LoadN mem));
1029 predicate(UseShenandoahGC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
1030 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1031 ins_cost(INSN_COST * 4);
1032 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
1033 ins_encode %{
1034 assert_different_registers($dst$$Register, $tmp$$Register);
1035 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
1036
1037 // Loads compressed pointer
1038 __ ldrw($dst$$Register, $mem$$Register);
1039
1040 // Loads gc_state
1041 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1042 __ ldrb($tmp$$Register, gcs_addr);
1043
1044 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
1045 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
1046 noreg /* addr */,
1047 $dst$$Register /* pre_val, in this case it'll be only used in the slowpath as tmp. */,
1048 $tmp$$Register /* gc_state */,
1049 true /* encoded_preval */);
1050 }
1051
1052 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
1053 $dst$$Register /* obj */,
1054 $mem$$Register /* addr */,
1055 true /* narrow */,
1056 maybe_null,
1057 $tmp$$Register /* gc_state */);
1058 %}
1059 ins_pipe(pipe_class_memory);
1060 %}
1061
1062 instruct loadN_volatile_shenandoah(iRegNNoSp dst, indirect mem, iRegNNoSp tmp, rFlagsReg cr)
1063 %{
1064 match(Set dst (LoadN mem));
1065 predicate(UseShenandoahGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
1066 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1067 ins_cost(INSN_COST * 3);
1068 format %{ "ldarw $dst, $mem\t# ptr"
1069 "decode_heap_oop $dst, $dst"
1070 %}
1071 ins_encode %{
1072 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
1073
1074 // Loads compressed pointer
1075 __ ldarw($dst$$Register, $mem$$Register);
1076
1077 // Loads gc_state
1078 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1079 __ ldrb($tmp$$Register, gcs_addr);
1080
1081 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
1082 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
1083 noreg /* obj */,
1084 $dst$$Register /* pre_val, in this case it'll be only used in the slowpath as tmp. */,
1085 $tmp$$Register /* gc_state */,
1086 true /* encoded_preval */);
1087 }
1088
1089 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
1090 $dst$$Register /* obj */,
1091 $mem$$Register /* addr */,
1092 true /* narrow */,
1093 maybe_null,
1094 $tmp$$Register /* gc_state */);
1095 %}
1096 ins_pipe(pipe_class_memory);
1097 %}
|