5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 source_hpp %{
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 %}
29
30 encode %{
31 enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
32 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
33 Register tmp = $tmp$$Register;
34 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
35 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
36 /*acquire*/ false, /*release*/ true, /*is_cae*/ false, $res$$Register);
37 %}
38
39 enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
40 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
41 Register tmp = $tmp$$Register;
42 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
43 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
44 /*acquire*/ true, /*release*/ true, /*is_cae*/ false, $res$$Register);
45 %}
46 %}
47
48 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
49
50 match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
51 ins_cost(2 * VOLATILE_REF_COST);
52
53 effect(TEMP tmp, KILL cr);
54
55 format %{
56 "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
57 %}
58
59 ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp, res));
60
61 ins_pipe(pipe_slow);
62 %}
63
64 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
65
66 match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
67 ins_cost(2 * VOLATILE_REF_COST);
68
69 effect(TEMP tmp, KILL cr);
70
71 format %{
72 "cmpxchgw_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
73 %}
74
75 ins_encode %{
76 Register tmp = $tmp$$Register;
77 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
78 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*is_cae*/ false, $res$$Register);
79 %}
80
81 ins_pipe(pipe_slow);
82 %}
83
84 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
85
86 predicate(needs_acquiring_load_exclusive(n));
87 match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
88 ins_cost(VOLATILE_REF_COST);
89
90 effect(TEMP tmp, KILL cr);
91
92 format %{
93 "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
94 %}
95
96 ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp, res));
97
98 ins_pipe(pipe_slow);
99 %}
100
101 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
102
103 predicate(needs_acquiring_load_exclusive(n));
104 match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
105 ins_cost(VOLATILE_REF_COST);
106
107 effect(TEMP tmp, KILL cr);
108
109 format %{
110 "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
111 %}
112
113 ins_encode %{
114 Register tmp = $tmp$$Register;
115 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
116 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*is_cae*/ false, $res$$Register);
117 %}
118
119 ins_pipe(pipe_slow);
120 %}
121
122 instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
123 match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval)));
124 ins_cost(2 * VOLATILE_REF_COST);
125 effect(TEMP_DEF res, TEMP tmp, KILL cr);
126 format %{
127 "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
128 %}
129 ins_encode %{
130 Register tmp = $tmp$$Register;
131 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
132 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
133 /*acquire*/ false, /*release*/ true, /*is_cae*/ true, $res$$Register);
134 %}
135 ins_pipe(pipe_slow);
136 %}
137
138 instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
139 match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval)));
140 ins_cost(2 * VOLATILE_REF_COST);
141 effect(TEMP_DEF res, TEMP tmp, KILL cr);
142 format %{
143 "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
144 %}
145 ins_encode %{
146 Register tmp = $tmp$$Register;
147 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
148 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
149 /*acquire*/ false, /*release*/ true, /*is_cae*/ true, $res$$Register);
150 %}
151 ins_pipe(pipe_slow);
152 %}
153
154 instruct compareAndExchangeNAcq_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
155 predicate(needs_acquiring_load_exclusive(n));
156 match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval)));
157 ins_cost(VOLATILE_REF_COST);
158 effect(TEMP_DEF res, TEMP tmp, KILL cr);
159 format %{
160 "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
161 %}
162 ins_encode %{
163 Register tmp = $tmp$$Register;
164 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
165 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
166 /*acquire*/ true, /*release*/ true, /*is_cae*/ true, $res$$Register);
167 %}
168 ins_pipe(pipe_slow);
169 %}
170
171 instruct compareAndExchangePAcq_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
172 predicate(needs_acquiring_load_exclusive(n));
173 match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval)));
174 ins_cost(VOLATILE_REF_COST);
175 effect(TEMP_DEF res, TEMP tmp, KILL cr);
176 format %{
177 "cmpxchg_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
178 %}
179 ins_encode %{
180 Register tmp = $tmp$$Register;
181 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
182 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
183 /*acquire*/ true, /*release*/ true, /*is_cae*/ true, $res$$Register);
184 %}
185 ins_pipe(pipe_slow);
186 %}
187
188 instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
189 match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval)));
190 ins_cost(2 * VOLATILE_REF_COST);
191 effect(TEMP tmp, KILL cr);
192 format %{
193 "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
194 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
195 %}
196 ins_encode %{
197 Register tmp = $tmp$$Register;
198 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
199 // Weak is not currently supported by ShenandoahBarrierSet::cmpxchg_oop
200 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
201 /*acquire*/ false, /*release*/ true, /*is_cae*/ false, $res$$Register);
202 %}
203 ins_pipe(pipe_slow);
204 %}
205
206 instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
207 match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval)));
208 ins_cost(2 * VOLATILE_REF_COST);
209 effect(TEMP tmp, KILL cr);
210 format %{
211 "cmpxchg_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
212 %}
213 ins_encode %{
214 Register tmp = $tmp$$Register;
215 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
216 // Weak is not currently supported by ShenandoahBarrierSet::cmpxchg_oop
217 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
218 /*acquire*/ false, /*release*/ true, /*is_cae*/ false, $res$$Register);
219 %}
220 ins_pipe(pipe_slow);
221 %}
222
223 instruct weakCompareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
224 predicate(needs_acquiring_load_exclusive(n));
225 match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval)));
226 ins_cost(VOLATILE_REF_COST);
227 effect(TEMP tmp, KILL cr);
228 format %{
229 "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
230 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
231 %}
232 ins_encode %{
233 Register tmp = $tmp$$Register;
234 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
235 // Weak is not currently supported by ShenandoahBarrierSet::cmpxchg_oop
236 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
237 /*acquire*/ true, /*release*/ true, /*is_cae*/ false, $res$$Register);
238 %}
239 ins_pipe(pipe_slow);
240 %}
241
242 instruct weakCompareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
243 predicate(needs_acquiring_load_exclusive(n));
244 match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval)));
245 ins_cost(VOLATILE_REF_COST);
246 effect(TEMP tmp, KILL cr);
247 format %{
248 "cmpxchg_acq_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
249 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
250 %}
251 ins_encode %{
252 Register tmp = $tmp$$Register;
253 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
254 // Weak is not currently supported by ShenandoahBarrierSet::cmpxchg_oop
255 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, $mem$$Register, tmp, $newval$$Register,
256 /*acquire*/ true, /*release*/ true, /*is_cae*/ false, $res$$Register);
257 %}
258 ins_pipe(pipe_slow);
259 %}
|
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 source %{
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 %}
30
31 instruct storeP_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
32 %{
33 match(Set mem (StoreP mem src));
34 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
35 effect(TEMP tmp, KILL cr);
36 ins_cost(INSN_COST);
37 format %{ "str $src, $mem\t# ptr" %}
38 ins_encode %{
39 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
40 $mem$$Register /* addr */,
41 $tmp$$Register /* pre_val */);
42 __ str($src$$Register, $mem$$Register);
43 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
44 $mem$$Register /* addr */,
45 $tmp$$Register /* tmp */);
46 %}
47 ins_pipe(istore_reg_mem);
48 %}
49
50 instruct storePVolatile_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
51 %{
52 match(Set mem (StoreP mem src));
53 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
54 effect(TEMP tmp, KILL cr);
55 ins_cost(VOLATILE_REF_COST);
56 format %{ "stlr $src, $mem\t# ptr" %}
57 ins_encode %{
58 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
59 $mem$$Register /* obj */,
60 $tmp$$Register /* pre_val */);
61 __ stlr($src$$Register, $mem$$Register);
62 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
63 $mem$$Register /* addr */,
64 $tmp$$Register /* tmp */);
65 %}
66 ins_pipe(pipe_class_memory);
67 %}
68
69 instruct storeN_shenandoah(indirect mem, iRegN src, iRegPNoSp tmp, rFlagsReg cr)
70 %{
71 match(Set mem (StoreN mem src));
72 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
73 effect(TEMP tmp, KILL cr);
74 ins_cost(INSN_COST);
75 format %{ "strw $src, $mem\t# compressed ptr" %}
76 ins_encode %{
77 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
78 $mem$$Register /* obj */,
79 $tmp$$Register /* pre_val */);
80 __ strw($src$$Register, $mem$$Register);
81 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
82 $mem$$Register /* addr */,
83 $tmp$$Register /* tmp */);
84 %}
85 ins_pipe(istore_reg_mem);
86 %}
87
88 instruct storeNVolatile_shenandoah(indirect mem, iRegN src, iRegPNoSp tmp, rFlagsReg cr)
89 %{
90 match(Set mem (StoreN mem src));
91 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
92 effect(TEMP tmp, KILL cr);
93 ins_cost(VOLATILE_REF_COST);
94 format %{ "stlrw $src, $mem\t# compressed ptr" %}
95 ins_encode %{
96 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
97 $mem$$Register /* obj */,
98 $tmp$$Register /* pre_val */);
99 __ stlrw($src$$Register, $mem$$Register);
100 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
101 $mem$$Register /* addr */,
102 $tmp$$Register /* tmp */);
103 %}
104 ins_pipe(pipe_class_memory);
105 %}
106
107 instruct encodePAndStoreN_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
108 %{
109 match(Set mem (StoreN mem (EncodeP src)));
110 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
111 effect(TEMP tmp, KILL cr);
112 ins_cost(INSN_COST);
113 format %{ "encode_heap_oop $tmp, $src\n\t"
114 "strw $tmp, $mem\t# compressed ptr" %}
115 ins_encode %{
116 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
117 $mem$$Register /* obj */,
118 $tmp$$Register /* pre_val */);
119 if ((barrier_data() & ShenandoahBarrierCardMarkNotNull) == 0) {
120 __ encode_heap_oop($tmp$$Register, $src$$Register);
121 } else {
122 __ encode_heap_oop_not_null($tmp$$Register, $src$$Register);
123 }
124 __ strw($tmp$$Register, $mem$$Register);
125 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
126 $mem$$Register /* addr */,
127 $tmp$$Register /* tmp */);
128 %}
129 ins_pipe(istore_reg_mem);
130 %}
131
132 instruct encodePAndStoreNVolatile_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
133 %{
134 match(Set mem (StoreN mem (EncodeP src)));
135 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
136 effect(TEMP tmp, KILL cr);
137 ins_cost(VOLATILE_REF_COST);
138 format %{ "encode_heap_oop $tmp, $src\n\t"
139 "stlrw $tmp, $mem\t# compressed ptr" %}
140 ins_encode %{
141 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
142 $mem$$Register /* obj */,
143 $tmp$$Register /* pre_val */);
144 if ((barrier_data() & ShenandoahBarrierCardMarkNotNull) == 0) {
145 __ encode_heap_oop($tmp$$Register, $src$$Register);
146 } else {
147 __ encode_heap_oop_not_null($tmp$$Register, $src$$Register);
148 }
149 __ stlrw($tmp$$Register, $mem$$Register);
150 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
151 $mem$$Register /* addr */,
152 $tmp$$Register /* tmp */);
153 %}
154 ins_pipe(pipe_class_memory);
155 %}
156
157 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
158
159 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
160 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
161 ins_cost(2 * VOLATILE_REF_COST);
162
163 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
164
165 format %{
166 "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
167 %}
168
169 ins_encode %{
170 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
171 guarantee(!UseCompressedOops, "must not be compressed oops");
172
173 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
174 __ mov($tmp1$$Register, $oldval$$Register);
175 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
176 noreg /* addr */,
177 $tmp1$$Register /* pre_val */);
178 }
179
180 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
181 $mem$$base$$Register,
182 $oldval$$Register,
183 $newval$$Register,
184 $res$$Register,
185 $tmp1$$Register,
186 $tmp2$$Register,
187 /*acquire*/ false,
188 /*release*/ true,
189 /*weak*/ false,
190 /*is_cae*/ false);
191
192 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
193 $mem$$Register /* addr */,
194 $tmp1$$Register /* tmp */);
195 %}
196
197 ins_pipe(pipe_slow);
198 %}
199
200 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
201
202 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
203 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
204 ins_cost(2 * VOLATILE_REF_COST);
205
206 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
207
208 format %{
209 "cmpxchgw_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
210 %}
211
212 ins_encode %{
213 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
214 guarantee(UseCompressedOops, "must be compressed oops");
215 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
216 __ decode_heap_oop($tmp1$$Register, $oldval$$Register);
217 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
218 noreg /* obj */,
219 $tmp1$$Register /* pre_val */);
220 }
221 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
222 $mem$$base$$Register,
223 $oldval$$Register,
224 $newval$$Register,
225 $res$$Register,
226 $tmp1$$Register,
227 $tmp2$$Register,
228 /*acquire*/ false,
229 /*release*/ true,
230 /*weak*/ false,
231 /*is_cae*/ false);
232
233 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
234 $mem$$Register /* addr */,
235 $tmp1$$Register /* tmp */);
236 %}
237
238 ins_pipe(pipe_slow);
239 %}
240
241 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegINoSp tmp1, iRegINoSp tmp2, rFlagsReg cr) %{
242
243 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
244 predicate(needs_acquiring_load_exclusive(n));
245 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
246 ins_cost(VOLATILE_REF_COST);
247
248 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
249
250 format %{
251 "cmpxchg_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
252 %}
253
254 ins_encode %{
255 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
256 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
257 noreg /* obj */,
258 $oldval$$Register /* pre_val */);
259
260 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
261 $mem$$base$$Register,
262 $oldval$$Register,
263 $newval$$Register,
264 $res$$Register,
265 $tmp1$$Register,
266 $tmp2$$Register,
267 /*acquire*/ true,
268 /*release*/ true,
269 /*weak*/ false,
270 /*is_cae*/ false);
271
272 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
273 $mem$$Register /* addr */,
274 $tmp1$$Register /* tmp */);
275 %}
276
277 ins_pipe(pipe_slow);
278 %}
279
280 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
281
282 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
283 predicate(needs_acquiring_load_exclusive(n));
284 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
285 ins_cost(VOLATILE_REF_COST);
286
287 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
288
289 format %{
290 "cmpxchgw_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
291 %}
292
293 ins_encode %{
294 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
295 guarantee(UseCompressedOops, "must be compressed oops");
296 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
297 __ decode_heap_oop($tmp1$$Register, $oldval$$Register);
298 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
299 noreg /* obj */,
300 $tmp1$$Register /* pre_val */);
301 }
302 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
303 $mem$$base$$Register,
304 $oldval$$Register,
305 $newval$$Register,
306 $res$$Register,
307 $tmp1$$Register,
308 $tmp2$$Register,
309 /*acquire*/ true,
310 /*release*/ true,
311 /*weak*/ false,
312 /*is_cae*/ false);
313
314 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
315 $mem$$Register /* addr */,
316 $tmp1$$Register /* tmp */);
317 %}
318
319 ins_pipe(pipe_slow);
320 %}
321
322 instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
323 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
324 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0 && !needs_acquiring_load_exclusive(n));
325 ins_cost(2 * VOLATILE_REF_COST);
326 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
327 format %{
328 "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
329 %}
330 ins_encode %{
331 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
332 guarantee(UseCompressedOops, "must be compressed oops");
333 __ block_comment("compareAndExchangeN_shenandoah");
334 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
335 __ decode_heap_oop($tmp1$$Register, $oldval$$Register);
336 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm, noreg /* obj */, $tmp1$$Register /* pre_val */);
337 }
338 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
339 $mem$$base$$Register,
340 $oldval$$Register,
341 $newval$$Register,
342 $res$$Register,
343 $tmp1$$Register,
344 $tmp2$$Register,
345 /*acquire*/ false,
346 /*release*/ true,
347 /*weak*/ false,
348 /*is_cae*/ true);
349
350 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
351 $res$$Register /* obj */,
352 $mem$$Register /* addr */,
353 $tmp1$$Register /* tmp */,
354 true /* narrow */,
355 true /* maybe_null */);
356
357 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
358 $mem$$Register /* addr */,
359 $tmp1$$Register /* tmp */);
360 __ block_comment("} compareAndExchangeN_shenandoah");
361 %}
362 ins_pipe(pipe_slow);
363 %}
364
365 instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
366 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
367 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
368 ins_cost(2 * VOLATILE_REF_COST);
369 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
370 format %{
371 "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
372 %}
373 ins_encode %{
374 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
375 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
376 noreg /* obj */,
377 $oldval$$Register /* pre_val */);
378
379 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
380 $mem$$base$$Register,
381 $oldval$$Register,
382 $newval$$Register,
383 $res$$Register,
384 $tmp1$$Register,
385 $tmp2$$Register,
386 /*acquire*/ false,
387 /*release*/ true,
388 /*weak*/ false,
389 /*is_cae*/ true);
390
391 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
392 $res$$Register /* obj */,
393 $mem$$Register /* addr */,
394 $tmp1$$Register /* tmp */,
395 false /* narrow */,
396 true /* maybe_null */);
397 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
398 $mem$$Register /* addr */,
399 $tmp1$$Register /* tmp */);
400 %}
401 ins_pipe(pipe_slow);
402 %}
403
404 instruct compareAndExchangeNAcq_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
405 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
406 predicate(needs_acquiring_load_exclusive(n));
407 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
408 ins_cost(VOLATILE_REF_COST);
409 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
410 format %{
411 "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
412 %}
413 ins_encode %{
414 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
415 guarantee(UseCompressedOops, "must be compressed oops");
416 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
417 __ decode_heap_oop($tmp1$$Register, $oldval$$Register);
418 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
419 noreg /* obj */,
420 $tmp1$$Register /* pre_val */);
421 }
422
423 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
424 $mem$$base$$Register,
425 $oldval$$Register,
426 $newval$$Register,
427 $res$$Register,
428 $tmp1$$Register,
429 $tmp2$$Register,
430 /*acquire*/ true,
431 /*release*/ true,
432 /*weak*/ false,
433 /*is_cae*/ true);
434
435 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
436 $res$$Register /* obj */,
437 $mem$$Register /* addr */,
438 $tmp1$$Register /* tmp */,
439 true /* narrow */,
440 true /* maybe_null */);
441 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
442 $mem$$Register /* addr */,
443 $tmp1$$Register /* tmp */);
444 %}
445 ins_pipe(pipe_slow);
446 %}
447
448 instruct compareAndExchangePAcq_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
449 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
450 predicate(needs_acquiring_load_exclusive(n));
451 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
452 ins_cost(VOLATILE_REF_COST);
453 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
454 format %{
455 "cmpxchg_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
456 %}
457 ins_encode %{
458 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
459 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
460 noreg /* obj */,
461 $oldval$$Register /* pre_val */);
462
463 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
464 $mem$$base$$Register,
465 $oldval$$Register,
466 $newval$$Register,
467 $res$$Register,
468 $tmp1$$Register,
469 $tmp2$$Register,
470 /*acquire*/ true,
471 /*release*/ true,
472 /*weak*/ false,
473 /*is_cae*/ true);
474
475 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
476 $res$$Register /* obj */,
477 $mem$$Register /* addr */,
478 $tmp1$$Register /* tmp */,
479 false /* narrow */,
480 true /* maybe_null */);
481 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
482 $mem$$Register /* addr */,
483 $tmp1$$Register /* tmp */);
484 %}
485 ins_pipe(pipe_slow);
486 %}
487
488 instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
489 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
490 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
491 ins_cost(2 * VOLATILE_REF_COST);
492 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
493 format %{
494 "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
495 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
496 %}
497 ins_encode %{
498 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
499 guarantee(UseCompressedOops, "must be compressed oops");
500 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
501 __ decode_heap_oop($tmp1$$Register, $oldval$$Register);
502 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
503 noreg /* obj */,
504 $tmp1$$Register /* pre_val */);
505 }
506
507 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
508 $mem$$base$$Register,
509 $oldval$$Register,
510 $newval$$Register,
511 $res$$Register,
512 $tmp1$$Register,
513 $tmp2$$Register,
514 /*acquire*/ false,
515 /*release*/ true,
516 /*weak*/ true,
517 /*is_cae*/ false);
518
519 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
520 $mem$$Register /* addr */,
521 $tmp1$$Register /* tmp */);
522 %}
523 ins_pipe(pipe_slow);
524 %}
525
526 instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegINoSp tmp1, iRegINoSp tmp2, rFlagsReg cr) %{
527 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
528 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
529 ins_cost(2 * VOLATILE_REF_COST);
530 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
531 format %{
532 "cmpxchg_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
533 %}
534 ins_encode %{
535 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
536 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
537 noreg /* obj */,
538 $oldval$$Register /* pre_val */);
539
540 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
541 $mem$$base$$Register,
542 $oldval$$Register,
543 $newval$$Register,
544 $res$$Register,
545 $tmp1$$Register,
546 $tmp2$$Register,
547 /*acquire*/ false,
548 /*release*/ true,
549 /*weak*/ true,
550 /*is_cae*/ false);
551
552 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
553 $mem$$Register /* addr */,
554 $tmp1$$Register /* tmp */);
555 %}
556 ins_pipe(pipe_slow);
557 %}
558
559 instruct weakCompareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
560 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
561 predicate(needs_acquiring_load_exclusive(n));
562 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
563 ins_cost(VOLATILE_REF_COST);
564 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
565 format %{
566 "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
567 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
568 %}
569 ins_encode %{
570 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
571 guarantee(UseCompressedOops, "must be compressed oops");
572 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
573 __ decode_heap_oop($tmp1$$Register, $oldval$$Register);
574 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
575 noreg /* obj */,
576 $tmp1$$Register /* pre_val */);
577 }
578
579 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
580 $mem$$base$$Register,
581 $oldval$$Register,
582 $newval$$Register,
583 $res$$Register,
584 $tmp1$$Register,
585 $tmp2$$Register,
586 /*acquire*/ true,
587 /*release*/ true,
588 /*weak*/ true,
589 /*is_cae*/ false);
590
591 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
592 $mem$$Register /* addr */,
593 $tmp1$$Register /* tmp */);
594 %}
595 ins_pipe(pipe_slow);
596 %}
597
598 instruct weakCompareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
599 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
600 predicate(needs_acquiring_load_exclusive(n));
601 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
602 ins_cost(VOLATILE_REF_COST);
603 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
604 format %{
605 "cmpxchg_acq_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
606 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
607 %}
608 ins_encode %{
609 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
610 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
611 noreg /* obj */,
612 $oldval$$Register /* pre_val */);
613
614 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
615 $mem$$base$$Register,
616 $oldval$$Register,
617 $newval$$Register,
618 $res$$Register,
619 $tmp1$$Register,
620 $tmp2$$Register,
621 /*acquire*/ true,
622 /*release*/ true,
623 /*weak*/ true,
624 /*is_cae*/ false);
625
626 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
627 $mem$$Register /* addr */,
628 $tmp1$$Register /* tmp */);
629 %}
630 ins_pipe(pipe_slow);
631 %}
632
633 instruct getAndSetP_shenandoah(indirect mem, iRegP newval, iRegPNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
634 %{
635 match(Set preval (GetAndSetP mem newval));
636 predicate(UseShenandoahGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
637 effect(TEMP_DEF preval, TEMP tmp, KILL cr);
638 ins_cost(2 * VOLATILE_REF_COST);
639 format %{ "atomic_xchg $preval, $newval, [$mem]" %}
640 ins_encode %{
641 assert_different_registers($mem$$Register, $newval$$Register);
642 __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
643 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
644 noreg /* obj */,
645 $preval$$Register /* pre_val */);
646 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
647 $preval$$Register /* obj */,
648 $mem$$Register /* addr */,
649 $tmp$$Register /* tmp */,
650 false /* narrow */,
651 true /* maybe_null */);
652 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
653 $mem$$Register /* addr */,
654 $tmp$$Register /* tmp */);
655 %}
656 ins_pipe(pipe_serial);
657 %}
658
659 instruct getAndSetPAcq_shenandoah(indirect mem, iRegP newval, iRegPNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
660 %{
661 match(Set preval (GetAndSetP mem newval));
662 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
663 effect(TEMP_DEF preval, TEMP tmp, KILL cr);
664 ins_cost(VOLATILE_REF_COST);
665 format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %}
666 ins_encode %{
667 assert_different_registers($mem$$Register, $newval$$Register);
668 __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
669 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
670 noreg /* obj */,
671 $preval$$Register /* pre_val */);
672 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
673 $preval$$Register /* obj */,
674 $mem$$Register /* addr */,
675 $tmp$$Register /* tmp */,
676 false /* narrow */,
677 true /* maybe_null */);
678 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
679 $mem$$Register /* addr */,
680 $tmp$$Register /* tmp */);
681 %}
682 ins_pipe(pipe_serial);
683 %}
684
685 instruct getAndSetN_shenandoah(indirect mem, iRegN newval, iRegNNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
686 %{
687 match(Set preval (GetAndSetN mem newval));
688 predicate(UseShenandoahGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
689 effect(TEMP preval, TEMP tmp, KILL cr);
690 ins_cost(2 * VOLATILE_REF_COST);
691 format %{ "atomic_xchgw $preval, $newval, [$mem]" %}
692 ins_encode %{
693 assert_different_registers($mem$$Register, $newval$$Register);
694 __ atomic_xchgw($preval$$Register, $newval$$Register, $mem$$Register);
695 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
696 __ decode_heap_oop($tmp$$Register, $preval$$Register);
697 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
698 noreg /* obj */,
699 $tmp$$Register /* pre_val */);
700 }
701 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
702 $preval$$Register /* obj */,
703 $mem$$Register /* addr */,
704 $tmp$$Register /* tmp */,
705 true /* narrow */,
706 true /* maybe_null */);
707 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
708 $mem$$Register /* addr */,
709 $tmp$$Register /* tmp */);
710 %}
711 ins_pipe(pipe_serial);
712 %}
713
714 instruct getAndSetNAcq_shenandoah(indirect mem, iRegN newval, iRegNNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
715 %{
716 match(Set preval (GetAndSetN mem newval));
717 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
718 effect(TEMP preval, TEMP tmp, KILL cr);
719 ins_cost(VOLATILE_REF_COST);
720 format %{ "atomic_xchgw_acq $preval, $newval, [$mem]" %}
721 ins_encode %{
722 assert_different_registers($mem$$Register, $newval$$Register);
723 __ atomic_xchgalw($preval$$Register, $newval$$Register, $mem$$Register);
724 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
725 __ decode_heap_oop($tmp$$Register, $preval$$Register);
726 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
727 noreg /* obj */,
728 $tmp$$Register /* pre_val */);
729 }
730 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
731 $preval$$Register /* obj */,
732 $mem$$Register /* addr */,
733 $tmp$$Register /* tmp */,
734 true /* narrow */,
735 true /* maybe_null */);
736 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
737 $mem$$Register /* addr */,
738 $tmp$$Register /* tmp */);
739 %}
740 ins_pipe(pipe_serial);
741 %}
742
743 instruct loadP_shenandoah(iRegPNoSp dst, indirect mem, iRegPNoSp tmp, rFlagsReg cr)
744 %{
745 // This instruction does not need an acquiring counterpart because it is only
746 // used for reference loading (Reference::get()).
747 match(Set dst (LoadP mem));
748 predicate(UseShenandoahGC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
749 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
750 ins_cost(4 * INSN_COST);
751 format %{ "ldr $dst, $mem\t# ptr" %}
752 ins_encode %{
753 __ ldr($dst$$Register, $mem$$Register);
754 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
755 noreg /* obj */,
756 $dst$$Register /* pre_val */);
757
758 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
759
760 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
761 $dst$$Register /* obj */,
762 $mem$$Register /* addr */,
763 $tmp$$Register /* tmp */,
764 false /* narrow */,
765 maybe_null);
766 %}
767 ins_pipe(iload_reg_mem);
768 %}
769
770 instruct loadP_volatile_shenandoah(iRegPNoSp dst, indirect mem, rFlagsReg cr)
771 %{
772 // This instruction does not need an acquiring counterpart because it is only
773 // used for reference loading (Reference::get()).
774 match(Set dst (LoadP mem));
775 predicate(UseShenandoahGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
776 effect(TEMP dst, KILL cr);
777 ins_cost(4 * INSN_COST);
778 format %{ "ldar $dst, $mem\t# ptr" %}
779 ins_encode %{
780 __ ldar($dst$$Register, $mem$$Register);
781 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
782 noreg /* obj */,
783 $dst$$Register /* pre_val */);
784
785 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
786 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
787 $dst$$Register /* obj */,
788 $mem$$Register /* addr */,
789 noreg /* noreg - not needed */,
790 false /* narrow */,
791 maybe_null);
792 %}
793 ins_pipe(iload_reg_mem);
794 %}
795
796 instruct loadN_decodeN_shenandoah(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
797 match(Set dst (DecodeN (LoadN mem)));
798 predicate(UseShenandoahGC && !needs_acquiring_load(n->in(1)) && n->in(1)->as_Load()->barrier_data() != 0);
799 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, KILL cr);
800 ins_cost(INSN_COST * 2);
801 format %{ "ldrw $dst, $mem\t# decode(load(n))" %}
802 ins_encode %{
803 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
804
805 // Loads compressed pointer
806 __ ldrw($dst$$Register, $mem$$Register);
807
808 // Eagerly decode since both barriers below would decode it anyway
809 __ decode_heap_oop($tmp2$$Register, $dst$$Register);
810
811 // Loads gc_state
812 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
813 __ ldrb($tmp1$$Register, gc_state);
814
815 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
816 ShenandoahBarrierSet::assembler()->satb_barrier_c3(this, masm,
817 noreg /* obj */,
818 $tmp2$$Register /* pre_val */,
819 $tmp1$$Register /* gc_state */);
820 }
821
822 ShenandoahBarrierSet::assembler()->load_ref_barrier_c3(this, masm,
823 $dst$$Register /* obj */,
824 $mem$$Register /* addr */,
825 $tmp2$$Register /* tmp1 */,
826 true /* narrow */,
827 maybe_null,
828 $tmp1$$Register /* gc_state */);
829
830 __ decode_heap_oop($dst$$Register, $dst$$Register);
831 %}
832 ins_pipe(ialu_reg);
833 %}
834
835 instruct loadN_shenandoah(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{
836 match(Set dst (LoadN mem));
837 predicate(UseShenandoahGC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
838 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, KILL cr);
839 ins_cost(INSN_COST * 4);
840 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
841 ins_encode %{
842 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
843
844 // Loads compressed pointer
845 __ ldrw($dst$$Register, $mem$$Register);
846
847 // Eagerly decode since both barriers below would decode it anyway
848 __ decode_heap_oop($tmp1$$Register, $dst$$Register);
849
850 // Loads gc_state
851 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
852 __ ldrb($tmp2$$Register, gc_state);
853
854 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
855 ShenandoahBarrierSet::assembler()->satb_barrier_c3(this, masm,
856 noreg /* obj */,
857 $tmp1$$Register /* pre_val */,
858 $tmp2$$Register /* gc_state */);
859 }
860
861 ShenandoahBarrierSet::assembler()->load_ref_barrier_c3(this, masm,
862 $dst$$Register /* obj */,
863 $mem$$Register /* addr */,
864 $tmp1$$Register /* tmp1 */,
865 true /* narrow */,
866 maybe_null,
867 $tmp2$$Register /* gc_state */);
868 %}
869 ins_pipe(ialu_reg);
870 %}
871
872 instruct loadN_volatile_shenandoah(iRegNNoSp dst, indirect mem, iRegPNoSp tmp, rFlagsReg cr) %{
873 predicate(UseShenandoahGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
874 match(Set dst (LoadN mem));
875 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
876 ins_cost(INSN_COST * 3);
877 format %{ "ldarw $dst, $mem\t# ptr"
878 "decode_heap_oop $dst, $dst"
879 %}
880 ins_encode %{
881 __ ldarw($dst$$Register, $mem$$Register);
882 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
883 __ decode_heap_oop($tmp$$Register, $dst$$Register);
884 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
885 noreg /* obj */,
886 $tmp$$Register /* pre_val */);
887 }
888
889 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
890 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
891 $dst$$Register /* obj */,
892 $mem$$Register /* addr */,
893 $tmp$$Register /* tmp */,
894 true /* narrow */,
895 maybe_null);
896 %}
897 ins_pipe(ialu_reg);
898 %}
|