9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 source_hpp %{
27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
29 %}
30
31 // Weak compareAndSwap operations are treated as strong compareAndSwap operations.
32 // This is motivated by the retry logic of ShenandoahBarrierSetAssembler::cmpxchg_oop which is hard to realise
33 // using weak CAS operations.
34
35 instruct compareAndSwapP_shenandoah(iRegIdst res, indirect mem, iRegPsrc oldval, iRegPsrc newval,
36 iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr) %{
37 match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
38 match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval)));
39 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
40
41 predicate(((CompareAndSwapNode*)n)->order() != MemNode::acquire
42 && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst);
43
44 format %{ "CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %}
45 ins_encode %{
46 ShenandoahBarrierSet::assembler()->cmpxchg_oop(
47 masm,
48 $mem$$Register, $oldval$$Register, $newval$$Register,
49 $tmp1$$Register, $tmp2$$Register,
50 false, $res$$Register
51 );
52 %}
53 ins_pipe(pipe_class_default);
54 %}
55
56 instruct compareAndSwapN_shenandoah(iRegIdst res, indirect mem, iRegNsrc oldval, iRegNsrc newval,
57 iRegNdst tmp1, iRegNdst tmp2, flagsRegCR0 cr) %{
58 match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
59 match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval)));
60 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
61
62 predicate(((CompareAndSwapNode*)n)->order() != MemNode::acquire
63 && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst);
64
65 format %{ "CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %}
66 ins_encode %{
67 ShenandoahBarrierSet::assembler()->cmpxchg_oop(
68 masm,
69 $mem$$Register, $oldval$$Register, $newval$$Register,
70 $tmp1$$Register, $tmp2$$Register,
71 false, $res$$Register
72 );
73 %}
74 ins_pipe(pipe_class_default);
75 %}
76
77 instruct compareAndSwapP_acq_shenandoah(iRegIdst res, indirect mem, iRegPsrc oldval, iRegPsrc newval,
78 iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr) %{
79 match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
80 match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval)));
81 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
82
83 predicate(((CompareAndSwapNode*)n)->order() == MemNode::acquire
84 || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst);
85
86 format %{ "CMPXCHGD acq $res, $mem, $oldval, $newval; as bool; ptr" %}
87 ins_encode %{
88 ShenandoahBarrierSet::assembler()->cmpxchg_oop(
89 masm,
90 $mem$$Register, $oldval$$Register, $newval$$Register,
91 $tmp1$$Register, $tmp2$$Register,
92 false, $res$$Register
93 );
94 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
95 __ isync();
96 } else {
97 __ sync();
98 }
99 %}
100 ins_pipe(pipe_class_default);
101 %}
102
103 instruct compareAndSwapN_acq_shenandoah(iRegIdst res, indirect mem, iRegNsrc oldval, iRegNsrc newval,
104 iRegNdst tmp1, iRegNdst tmp2, flagsRegCR0 cr) %{
105 match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
106 match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval)));
107 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
108
109 predicate(((CompareAndSwapNode*)n)->order() == MemNode::acquire
110 || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst);
111
112 format %{ "CMPXCHGD acq $res, $mem, $oldval, $newval; as bool; ptr" %}
113 ins_encode %{
114 ShenandoahBarrierSet::assembler()->cmpxchg_oop(
115 masm,
116 $mem$$Register, $oldval$$Register, $newval$$Register,
117 $tmp1$$Register, $tmp2$$Register,
118 false, $res$$Register
119 );
120 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
121 __ isync();
122 } else {
123 __ sync();
124 }
125 %}
126 ins_pipe(pipe_class_default);
127 %}
128
129 instruct compareAndExchangeP_shenandoah(iRegPdst res, indirect mem, iRegPsrc oldval, iRegPsrc newval,
130 iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr) %{
131 match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval)));
132 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
133
134 predicate(((CompareAndSwapNode*)n)->order() != MemNode::acquire
135 && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst);
136
137 format %{ "CMPXCHGD $res, $mem, $oldval, $newval; as ptr; ptr" %}
138 ins_encode %{
139 ShenandoahBarrierSet::assembler()->cmpxchg_oop(
140 masm,
141 $mem$$Register, $oldval$$Register, $newval$$Register,
142 $tmp1$$Register, $tmp2$$Register,
143 true, $res$$Register
144 );
145 %}
146 ins_pipe(pipe_class_default);
147 %}
148
149 instruct compareAndExchangeN_shenandoah(iRegNdst res, indirect mem, iRegNsrc oldval, iRegNsrc newval,
150 iRegNdst tmp1, iRegNdst tmp2, flagsRegCR0 cr) %{
151 match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval)));
152 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
153
154 predicate(((CompareAndSwapNode*)n)->order() != MemNode::acquire
155 && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst);
156
157 format %{ "CMPXCHGD $res, $mem, $oldval, $newval; as ptr; ptr" %}
158 ins_encode %{
159 ShenandoahBarrierSet::assembler()->cmpxchg_oop(
160 masm,
161 $mem$$Register, $oldval$$Register, $newval$$Register,
162 $tmp1$$Register, $tmp2$$Register,
163 true, $res$$Register
164 );
165 %}
166 ins_pipe(pipe_class_default);
167 %}
168
169 instruct compareAndExchangePAcq_shenandoah(iRegPdst res, indirect mem, iRegPsrc oldval, iRegPsrc newval,
170 iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr) %{
171 match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval)));
172 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
173
174 predicate(((CompareAndSwapNode*)n)->order() == MemNode::acquire
175 || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst);
176
177 format %{ "CMPXCHGD acq $res, $mem, $oldval, $newval; as ptr; ptr" %}
178 ins_encode %{
179 ShenandoahBarrierSet::assembler()->cmpxchg_oop(
180 masm,
181 $mem$$Register, $oldval$$Register, $newval$$Register,
182 $tmp1$$Register, $tmp2$$Register,
183 true, $res$$Register
184 );
185 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
186 __ isync();
187 } else {
188 __ sync();
189 }
190 %}
191 ins_pipe(pipe_class_default);
192 %}
193
194 instruct compareAndExchangeNAcq_shenandoah(iRegNdst res, indirect mem, iRegNsrc oldval, iRegNsrc newval,
195 iRegNdst tmp1, iRegNdst tmp2, flagsRegCR0 cr) %{
196 match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval)));
197 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
198
199 predicate(((CompareAndSwapNode*)n)->order() == MemNode::acquire
200 || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst);
201
202 format %{ "CMPXCHGD acq $res, $mem, $oldval, $newval; as ptr; ptr" %}
203 ins_encode %{
204 ShenandoahBarrierSet::assembler()->cmpxchg_oop(
205 masm,
206 $mem$$Register, $oldval$$Register, $newval$$Register,
207 $tmp1$$Register, $tmp2$$Register,
208 true, $res$$Register
209 );
210 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
211 __ isync();
212 } else {
213 __ sync();
214 }
215 %}
216 ins_pipe(pipe_class_default);
217 %}
|
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 source_hpp %{
27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
29
30 bool need_acquire_load(const Node* n);
31 bool need_acquire_load_store(const Node *load);
32 %}
33
34 source %{
35 bool need_acquire_load(const Node* n) {
36 return !n->as_Load()->is_unordered() && !followed_by_acquire(n);
37 }
38 bool need_acquire_load_store(const Node* n) {
39 MemNode::MemOrd order = ((CompareAndSwapNode*)n->as_LoadStore())->order();
40 return (order == MemNode::acquire) || (order == MemNode::seqcst);
41 }
42 %}
43
44 // ---------------------------------- LOADS ---------------------------------------
45 //
46
47 instruct loadN_shenandoah(iRegNdst dst, memory mem) %{
48 match(Set dst (LoadN mem));
49 predicate(UseShenandoahGC && (n->as_Load()->barrier_data() != 0) &&
50 !need_acquire_load(n));
51 // The main load is a candidate to implement implicit null checks.
52 ins_is_late_expanded_null_check_candidate(true);
53 format %{ "shenandoah_load $dst, $mem\t# ptr" %}
54 ins_encode %{
55 ShenandoahBarrierSet::assembler()->load_c2(this, masm,
56 $dst$$Register,
57 $mem$$base$$Register,
58 $mem$$disp,
59 /* narrow = */ true,
60 /* acquire = */ false
61 );
62 %}
63 ins_cost(MEMORY_REF_COST);
64 ins_pipe(pipe_class_memory);
65 %}
66
67 instruct loadN_acq_shenandoah(iRegNdst dst, memory mem) %{
68 match(Set dst (LoadN mem));
69 predicate(UseShenandoahGC && (n->as_Load()->barrier_data() != 0) &&
70 need_acquire_load(n));
71 // The main load is a candidate to implement implicit null checks.
72 ins_is_late_expanded_null_check_candidate(true);
73 format %{ "shenandoah_load $dst, $mem\t# ptr (acquire)" %}
74 ins_encode %{
75 ShenandoahBarrierSet::assembler()->load_c2(this, masm,
76 $dst$$Register,
77 $mem$$base$$Register,
78 $mem$$disp,
79 /* narrow = */ true,
80 /* acquire = */ true
81 );
82 %}
83 ins_cost(3*MEMORY_REF_COST);
84 ins_pipe(pipe_class_memory);
85 %}
86
87 instruct loadP_shenandoah(iRegPdst dst, memoryAlg4 mem) %{
88 match(Set dst (LoadP mem));
89 predicate(UseShenandoahGC && (n->as_Load()->barrier_data() != 0) &&
90 !need_acquire_load(n));
91 // The main load is a candidate to implement implicit null checks.
92 ins_is_late_expanded_null_check_candidate(true);
93 format %{ "shenandoah_load $dst, $mem\t# ptr (acquire)" %}
94 ins_encode %{
95 ShenandoahBarrierSet::assembler()->load_c2(this, masm,
96 $dst$$Register,
97 $mem$$base$$Register,
98 $mem$$disp,
99 /* narrow = */ false,
100 /* acquire = */ false
101 );
102 %}
103 ins_cost(MEMORY_REF_COST);
104 ins_pipe(pipe_class_memory);
105 %}
106
107 instruct loadP_acq_shenandoah(iRegPdst dst, memoryAlg4 mem) %{
108 match(Set dst (LoadP mem));
109 predicate(UseShenandoahGC && (n->as_Load()->barrier_data() != 0) &&
110 need_acquire_load(n));
111 // The main load is a candidate to implement implicit null checks.
112 ins_is_late_expanded_null_check_candidate(true);
113 format %{ "shenandoah_load $dst, $mem\t# ptr (acquire)" %}
114 ins_encode %{
115 ShenandoahBarrierSet::assembler()->load_c2(this, masm,
116 $dst$$Register,
117 $mem$$base$$Register,
118 $mem$$disp,
119 /* narrow = */ false,
120 /* acquire = */ true
121 );
122 %}
123 ins_cost(3*MEMORY_REF_COST);
124 ins_pipe(pipe_class_memory);
125 %}
126
127 // ---------------------------------- STORES ---------------------------------------
128 //
129
130 instruct storeN_shenandoah(memory dst, iRegN_P2N src, iRegPdst tmp) %{
131 match(Set dst (StoreN dst src));
132 predicate(UseShenandoahGC && (n->as_Store()->barrier_data() != 0));
133 effect(TEMP tmp);
134 format %{ "shenandoah_store $dst, $src\t# compressed ptr" %}
135 ins_encode %{
136 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
137 $dst$$base$$Register,
138 $dst$$disp,
139 /* dst_narrow = */ true,
140 $src$$Register,
141 /* src_narrow = */ true,
142 $tmp$$Register
143 );
144 %}
145 ins_cost(MEMORY_REF_COST);
146 ins_pipe(pipe_class_memory);
147 %}
148
149 instruct storeP_shenandoah(memoryAlg4 dst, iRegPsrc src, iRegPdst tmp) %{
150 match(Set dst (StoreP dst src));
151 predicate(UseShenandoahGC && (n->as_Store()->barrier_data() != 0));
152 effect(TEMP tmp);
153 format %{ "shenandoah_store $dst, $src\t# ptr" %}
154 ins_encode %{
155 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
156 $dst$$base$$Register,
157 $dst$$disp,
158 /* dst_narrow = */ false,
159 $src$$Register,
160 /* src_narrow = */ false,
161 $tmp$$Register
162 );
163 %}
164 ins_cost(MEMORY_REF_COST);
165 ins_pipe(pipe_class_memory);
166 %}
167
168 instruct encodePAndStoreN_shenandoah(memory dst, iRegPsrc src, iRegPdst tmp) %{
169 match(Set dst (StoreN dst (EncodeP src)));
170 predicate(UseShenandoahGC && (n->as_Store()->barrier_data() != 0));
171 effect(TEMP tmp);
172 format %{ "shenandoah_store $dst, $src\t# compressed ptr (with encoding)" %}
173 ins_encode %{
174 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
175 $dst$$base$$Register,
176 $dst$$disp,
177 /* dst_narrow = */ true,
178 $src$$Register,
179 /* src_narrow = */ false,
180 $tmp$$Register
181 );
182 %}
183 ins_cost(MEMORY_REF_COST);
184 ins_pipe(pipe_class_memory);
185 %}
186
187 // ---------------------- LOAD-STORES -----------------------------------
188 //
189
190 instruct compareAndSwapN_regP_regN_regN_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
191 match(Set res (CompareAndSwapN mem_ptr (Binary src1 src2)));
192 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
193 !need_acquire_load_store(n));
194 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
195 format %{ "CMPXCHGW $res, $mem_ptr, $src1, $src2; as bool" %}
196 ins_encode %{
197 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
198 __ cmpxchgw(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
199 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
200 $res$$Register, nullptr, true);
201 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
202 __ isync();
203 } else {
204 __ sync();
205 }
206 %}
207 ins_pipe(pipe_class_default);
208 %}
209
210 instruct compareAndSwapN_acq_regP_regN_regN_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
211 match(Set res (CompareAndSwapN mem_ptr (Binary src1 src2)));
212 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
213 need_acquire_load_store(n));
214 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
215 format %{ "CMPXCHGW $res, $mem_ptr, $src1, $src2; as bool" %}
216 ins_encode %{
217 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
218 __ cmpxchgw(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
219 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
220 $res$$Register, nullptr, true);
221 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
222 __ isync();
223 } else {
224 __ sync();
225 }
226 %}
227 ins_pipe(pipe_class_default);
228 %}
229
230 instruct compareAndSwapP_regP_regP_regP_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
231 match(Set res (CompareAndSwapP mem_ptr (Binary src1 src2)));
232 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
233 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
234 !need_acquire_load_store(n));
235 format %{ "CMPXCHGD $res, $mem_ptr, $src1, $src2; as bool; ptr" %}
236 ins_encode %{
237 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
238 __ cmpxchgd(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
239 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
240 $res$$Register, nullptr, true);
241 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
242 __ isync();
243 } else {
244 __ sync();
245 }
246 %}
247 ins_pipe(pipe_class_default);
248 %}
249
250 instruct compareAndSwapP_acq_regP_regP_regP_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
251 match(Set res (CompareAndSwapP mem_ptr (Binary src1 src2)));
252 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
253 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
254 need_acquire_load_store(n));
255 format %{ "CMPXCHGD $res, $mem_ptr, $src1, $src2; as bool; ptr" %}
256 ins_encode %{
257 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
258 __ cmpxchgd(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
259 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
260 $res$$Register, nullptr, true);
261 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
262 __ isync();
263 } else {
264 __ sync();
265 }
266 %}
267 ins_pipe(pipe_class_default);
268 %}
269
270 instruct weakCompareAndSwapN_regP_regN_regN_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
271 match(Set res (WeakCompareAndSwapN mem_ptr (Binary src1 src2)));
272 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
273 !need_acquire_load_store(n));
274 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
275 format %{ "weak CMPXCHGW acq $res, $mem_ptr, $src1, $src2; as bool" %}
276 ins_encode %{
277 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
278 // Acquire only needed in successful case. Weak node is allowed to report unsuccessful in additional rare cases and
279 // value is never passed to caller.
280 __ cmpxchgw(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
281 support_IRIW_for_not_multiple_copy_atomic_cpu ? MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter,
282 MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
283 %}
284 ins_pipe(pipe_class_default);
285 %}
286
287 instruct weakCompareAndSwapN_acq_regP_regN_regN_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
288 match(Set res (WeakCompareAndSwapN mem_ptr (Binary src1 src2)));
289 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
290 need_acquire_load_store(n));
291 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
292 format %{ "weak CMPXCHGW acq $res, $mem_ptr, $src1, $src2; as bool" %}
293 ins_encode %{
294 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
295 // Acquire only needed in successful case. Weak node is allowed to report unsuccessful in additional rare cases and
296 // value is never passed to caller.
297 __ cmpxchgw(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
298 support_IRIW_for_not_multiple_copy_atomic_cpu ? MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter,
299 MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
300 %}
301 ins_pipe(pipe_class_default);
302 %}
303
304 instruct weakCompareAndSwapP_regP_regP_regP_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
305 match(Set res (WeakCompareAndSwapP mem_ptr (Binary src1 src2)));
306 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
307 !need_acquire_load_store(n));
308 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
309 format %{ "weak CMPXCHGD $res, $mem_ptr, $src1, $src2; as bool; ptr" %}
310 ins_encode %{
311 MemNode::MemOrd order = ((CompareAndSwapNode*)as_LoadStore())->order();
312 bool acquire = (order == MemNode::acquire) || (order == MemNode::seqcst);
313
314 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
315 __ cmpxchgd(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
316 MacroAssembler::MemBarNone,
317 MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
318 %}
319 ins_pipe(pipe_class_default);
320 %}
321
322 instruct weakCompareAndSwapP_acq_regP_regP_regP_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
323 match(Set res (WeakCompareAndSwapP mem_ptr (Binary src1 src2)));
324 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
325 need_acquire_load_store(n));
326 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
327 format %{ "weak CMPXCHGD $res, $mem_ptr, $src1, $src2; as bool; ptr" %}
328 ins_encode %{
329 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
330 __ cmpxchgd(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
331 MacroAssembler::MemBarNone,
332 MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
333 %}
334 ins_pipe(pipe_class_default);
335 %}
336
337 instruct compareAndExchangeN_regP_regN_regN_shenandoah(iRegNdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
338 match(Set res (CompareAndExchangeN mem_ptr (Binary src1 src2)));
339 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
340 !need_acquire_load_store(n));
341 effect(TEMP_DEF res, TEMP cr0);
342 format %{ "CMPXCHGW $res, $mem_ptr, $src1, $src2; as narrow oop" %}
343 ins_encode %{
344 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
345 __ cmpxchgw(CR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
346 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
347 noreg, nullptr, true);
348 %}
349 ins_pipe(pipe_class_default);
350 %}
351
352 instruct compareAndExchangeN_acq_regP_regN_regN_shenandoah(iRegNdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
353 match(Set res (CompareAndExchangeN mem_ptr (Binary src1 src2)));
354 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
355 need_acquire_load_store(n));
356 effect(TEMP_DEF res, TEMP cr0);
357 format %{ "CMPXCHGW $res, $mem_ptr, $src1, $src2; as narrow oop" %}
358 ins_encode %{
359 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
360 __ cmpxchgw(CR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
361 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
362 noreg, nullptr, true);
363
364 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
365 __ isync();
366 } else {
367 // isync would be sufficient in case of CompareAndExchangeAcquire, but we currently don't optimize for that.
368 __ sync();
369 }
370 %}
371 ins_pipe(pipe_class_default);
372 %}
373
374 instruct compareAndExchangeP_regP_regP_regP_shenandoah(iRegPdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
375 match(Set res (CompareAndExchangeP mem_ptr (Binary src1 src2)));
376 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
377 !need_acquire_load_store(n));
378 effect(TEMP_DEF res, TEMP cr0);
379 format %{ "CMPXCHGD $res, $mem_ptr, $src1, $src2; as ptr; ptr" %}
380 ins_encode %{
381 MemNode::MemOrd order = ((CompareAndSwapNode*)as_LoadStore())->order();
382 bool acquire = (order == MemNode::acquire) || (order == MemNode::seqcst);
383
384 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
385 __ cmpxchgd(CR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
386 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
387 noreg, nullptr, true);
388 %}
389 ins_pipe(pipe_class_default);
390 %}
391
392 instruct compareAndExchangeP_acq_regP_regP_regP_shenandoah(iRegPdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
393 match(Set res (CompareAndExchangeP mem_ptr (Binary src1 src2)));
394 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
395 need_acquire_load_store(n));
396 effect(TEMP_DEF res, TEMP cr0);
397 format %{ "CMPXCHGD $res, $mem_ptr, $src1, $src2; as ptr; ptr" %}
398 ins_encode %{
399 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
400 __ cmpxchgd(CR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
401 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
402 noreg, nullptr, true);
403
404 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
405 __ isync();
406 } else {
407 // isync would be sufficient in case of CompareAndExchangeAcquire, but we currently don't optimize for that.
408 __ sync();
409 }
410 %}
411 ins_pipe(pipe_class_default);
412 %}
413
414 instruct getAndSetP_shenandoah(iRegPdst res, iRegPdst mem_ptr, iRegPsrc src, flagsRegCR0 cr0) %{
415 match(Set res (GetAndSetP mem_ptr src));
416 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0));
417 effect(TEMP_DEF res, TEMP cr0);
418 format %{ "GetAndSetP $res, $mem_ptr, $src" %}
419 ins_encode %{
420 __ getandsetd($res$$Register, $src$$Register, $mem_ptr$$Register,
421 MacroAssembler::cmpxchgx_hint_atomic_update());
422 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
423 __ isync();
424 } else {
425 __ sync();
426 }
427 %}
428 ins_pipe(pipe_class_default);
429 %}
430
431 instruct getAndSetN_shenandoah(iRegNdst res, iRegPdst mem_ptr, iRegNsrc src, flagsRegCR0 cr0) %{
432 match(Set res (GetAndSetN mem_ptr src));
433 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0));
434 effect(TEMP_DEF res, TEMP cr0);
435 format %{ "GetAndSetN $res, $mem_ptr, $src" %}
436 ins_encode %{
437 __ getandsetw($res$$Register, $src$$Register, $mem_ptr$$Register,
438 MacroAssembler::cmpxchgx_hint_atomic_update());
439 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
440 __ isync();
441 } else {
442 __ sync();
443 }
444 %}
445 ins_pipe(pipe_class_default);
446 %}
447
|