1 //
2 // Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
3 // Copyright (c) 2012, 2021 SAP SE. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 source_hpp %{
27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
29
30 bool need_acquire_load(const Node* n);
31 bool need_acquire_load_store(const Node *load);
32 %}
33
34 source %{
35 bool need_acquire_load(const Node* n) {
36 return !n->as_Load()->is_unordered() && !followed_by_acquire(n);
37 }
38 bool need_acquire_load_store(const Node* n) {
39 MemNode::MemOrd order = ((CompareAndSwapNode*)n->as_LoadStore())->order();
40 return (order == MemNode::acquire) || (order == MemNode::seqcst);
41 }
42 %}
43
44 // ---------------------------------- LOADS ---------------------------------------
45 //
46
47 instruct loadN_shenandoah(iRegNdst dst, memory mem) %{
48 match(Set dst (LoadN mem));
49 predicate(UseShenandoahGC && (n->as_Load()->barrier_data() != 0) &&
50 !need_acquire_load(n));
51 // The main load is a candidate to implement implicit null checks.
52 ins_is_late_expanded_null_check_candidate(true);
53 format %{ "shenandoah_load $dst, $mem\t# ptr" %}
54 ins_encode %{
55 ShenandoahBarrierSet::assembler()->load_c2(this, masm,
56 $dst$$Register,
57 $mem$$base$$Register,
58 $mem$$disp,
59 /* narrow = */ true,
60 /* acquire = */ false
61 );
62 %}
63 ins_cost(MEMORY_REF_COST);
64 ins_pipe(pipe_class_memory);
65 %}
66
67 instruct loadN_acq_shenandoah(iRegNdst dst, memory mem) %{
68 match(Set dst (LoadN mem));
69 predicate(UseShenandoahGC && (n->as_Load()->barrier_data() != 0) &&
70 need_acquire_load(n));
71 // The main load is a candidate to implement implicit null checks.
72 ins_is_late_expanded_null_check_candidate(true);
73 format %{ "shenandoah_load $dst, $mem\t# ptr (acquire)" %}
74 ins_encode %{
75 ShenandoahBarrierSet::assembler()->load_c2(this, masm,
76 $dst$$Register,
77 $mem$$base$$Register,
78 $mem$$disp,
79 /* narrow = */ true,
80 /* acquire = */ true
81 );
82 %}
83 ins_cost(3*MEMORY_REF_COST);
84 ins_pipe(pipe_class_memory);
85 %}
86
87 instruct loadP_shenandoah(iRegPdst dst, memoryAlg4 mem) %{
88 match(Set dst (LoadP mem));
89 predicate(UseShenandoahGC && (n->as_Load()->barrier_data() != 0) &&
90 !need_acquire_load(n));
91 // The main load is a candidate to implement implicit null checks.
92 ins_is_late_expanded_null_check_candidate(true);
93 format %{ "shenandoah_load $dst, $mem\t# ptr (acquire)" %}
94 ins_encode %{
95 ShenandoahBarrierSet::assembler()->load_c2(this, masm,
96 $dst$$Register,
97 $mem$$base$$Register,
98 $mem$$disp,
99 /* narrow = */ false,
100 /* acquire = */ false
101 );
102 %}
103 ins_cost(MEMORY_REF_COST);
104 ins_pipe(pipe_class_memory);
105 %}
106
107 instruct loadP_acq_shenandoah(iRegPdst dst, memoryAlg4 mem) %{
108 match(Set dst (LoadP mem));
109 predicate(UseShenandoahGC && (n->as_Load()->barrier_data() != 0) &&
110 need_acquire_load(n));
111 // The main load is a candidate to implement implicit null checks.
112 ins_is_late_expanded_null_check_candidate(true);
113 format %{ "shenandoah_load $dst, $mem\t# ptr (acquire)" %}
114 ins_encode %{
115 ShenandoahBarrierSet::assembler()->load_c2(this, masm,
116 $dst$$Register,
117 $mem$$base$$Register,
118 $mem$$disp,
119 /* narrow = */ false,
120 /* acquire = */ true
121 );
122 %}
123 ins_cost(3*MEMORY_REF_COST);
124 ins_pipe(pipe_class_memory);
125 %}
126
127 // ---------------------------------- STORES ---------------------------------------
128 //
129
130 instruct storeN_shenandoah(memory dst, iRegN_P2N src, iRegPdst tmp) %{
131 match(Set dst (StoreN dst src));
132 predicate(UseShenandoahGC && (n->as_Store()->barrier_data() != 0));
133 effect(TEMP tmp);
134 format %{ "shenandoah_store $dst, $src\t# compressed ptr" %}
135 ins_encode %{
136 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
137 $dst$$base$$Register,
138 $dst$$disp,
139 /* dst_narrow = */ true,
140 $src$$Register,
141 /* src_narrow = */ true,
142 $tmp$$Register
143 );
144 %}
145 ins_cost(MEMORY_REF_COST);
146 ins_pipe(pipe_class_memory);
147 %}
148
149 instruct storeP_shenandoah(memoryAlg4 dst, iRegPsrc src, iRegPdst tmp) %{
150 match(Set dst (StoreP dst src));
151 predicate(UseShenandoahGC && (n->as_Store()->barrier_data() != 0));
152 effect(TEMP tmp);
153 format %{ "shenandoah_store $dst, $src\t# ptr" %}
154 ins_encode %{
155 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
156 $dst$$base$$Register,
157 $dst$$disp,
158 /* dst_narrow = */ false,
159 $src$$Register,
160 /* src_narrow = */ false,
161 $tmp$$Register
162 );
163 %}
164 ins_cost(MEMORY_REF_COST);
165 ins_pipe(pipe_class_memory);
166 %}
167
168 instruct encodePAndStoreN_shenandoah(memory dst, iRegPsrc src, iRegPdst tmp) %{
169 match(Set dst (StoreN dst (EncodeP src)));
170 predicate(UseShenandoahGC && (n->as_Store()->barrier_data() != 0));
171 effect(TEMP tmp);
172 format %{ "shenandoah_store $dst, $src\t# compressed ptr (with encoding)" %}
173 ins_encode %{
174 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
175 $dst$$base$$Register,
176 $dst$$disp,
177 /* dst_narrow = */ true,
178 $src$$Register,
179 /* src_narrow = */ false,
180 $tmp$$Register
181 );
182 %}
183 ins_cost(MEMORY_REF_COST);
184 ins_pipe(pipe_class_memory);
185 %}
186
187 // ---------------------- LOAD-STORES -----------------------------------
188 //
189
190 instruct compareAndSwapN_regP_regN_regN_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
191 match(Set res (CompareAndSwapN mem_ptr (Binary src1 src2)));
192 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
193 !need_acquire_load_store(n));
194 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
195 format %{ "CMPXCHGW $res, $mem_ptr, $src1, $src2; as bool" %}
196 ins_encode %{
197 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
198 __ cmpxchgw(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
199 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
200 $res$$Register, nullptr, true);
201 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
202 __ isync();
203 } else {
204 __ sync();
205 }
206 %}
207 ins_pipe(pipe_class_default);
208 %}
209
210 instruct compareAndSwapN_acq_regP_regN_regN_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
211 match(Set res (CompareAndSwapN mem_ptr (Binary src1 src2)));
212 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
213 need_acquire_load_store(n));
214 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
215 format %{ "CMPXCHGW $res, $mem_ptr, $src1, $src2; as bool" %}
216 ins_encode %{
217 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
218 __ cmpxchgw(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
219 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
220 $res$$Register, nullptr, true);
221 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
222 __ isync();
223 } else {
224 __ sync();
225 }
226 %}
227 ins_pipe(pipe_class_default);
228 %}
229
230 instruct compareAndSwapP_regP_regP_regP_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
231 match(Set res (CompareAndSwapP mem_ptr (Binary src1 src2)));
232 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
233 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
234 !need_acquire_load_store(n));
235 format %{ "CMPXCHGD $res, $mem_ptr, $src1, $src2; as bool; ptr" %}
236 ins_encode %{
237 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
238 __ cmpxchgd(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
239 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
240 $res$$Register, nullptr, true);
241 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
242 __ isync();
243 } else {
244 __ sync();
245 }
246 %}
247 ins_pipe(pipe_class_default);
248 %}
249
250 instruct compareAndSwapP_acq_regP_regP_regP_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
251 match(Set res (CompareAndSwapP mem_ptr (Binary src1 src2)));
252 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
253 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
254 need_acquire_load_store(n));
255 format %{ "CMPXCHGD $res, $mem_ptr, $src1, $src2; as bool; ptr" %}
256 ins_encode %{
257 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
258 __ cmpxchgd(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
259 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
260 $res$$Register, nullptr, true);
261 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
262 __ isync();
263 } else {
264 __ sync();
265 }
266 %}
267 ins_pipe(pipe_class_default);
268 %}
269
270 instruct weakCompareAndSwapN_regP_regN_regN_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
271 match(Set res (WeakCompareAndSwapN mem_ptr (Binary src1 src2)));
272 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
273 !need_acquire_load_store(n));
274 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
275 format %{ "weak CMPXCHGW acq $res, $mem_ptr, $src1, $src2; as bool" %}
276 ins_encode %{
277 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
278 // Acquire only needed in successful case. Weak node is allowed to report unsuccessful in additional rare cases and
279 // value is never passed to caller.
280 __ cmpxchgw(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
281 support_IRIW_for_not_multiple_copy_atomic_cpu ? MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter,
282 MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
283 %}
284 ins_pipe(pipe_class_default);
285 %}
286
287 instruct weakCompareAndSwapN_acq_regP_regN_regN_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
288 match(Set res (WeakCompareAndSwapN mem_ptr (Binary src1 src2)));
289 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
290 need_acquire_load_store(n));
291 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
292 format %{ "weak CMPXCHGW acq $res, $mem_ptr, $src1, $src2; as bool" %}
293 ins_encode %{
294 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
295 // Acquire only needed in successful case. Weak node is allowed to report unsuccessful in additional rare cases and
296 // value is never passed to caller.
297 __ cmpxchgw(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
298 support_IRIW_for_not_multiple_copy_atomic_cpu ? MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter,
299 MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
300 %}
301 ins_pipe(pipe_class_default);
302 %}
303
304 instruct weakCompareAndSwapP_regP_regP_regP_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
305 match(Set res (WeakCompareAndSwapP mem_ptr (Binary src1 src2)));
306 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
307 !need_acquire_load_store(n));
308 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
309 format %{ "weak CMPXCHGD $res, $mem_ptr, $src1, $src2; as bool; ptr" %}
310 ins_encode %{
311 MemNode::MemOrd order = ((CompareAndSwapNode*)as_LoadStore())->order();
312 bool acquire = (order == MemNode::acquire) || (order == MemNode::seqcst);
313
314 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
315 __ cmpxchgd(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
316 MacroAssembler::MemBarNone,
317 MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
318 %}
319 ins_pipe(pipe_class_default);
320 %}
321
322 instruct weakCompareAndSwapP_acq_regP_regP_regP_shenandoah(iRegIdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
323 match(Set res (WeakCompareAndSwapP mem_ptr (Binary src1 src2)));
324 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
325 need_acquire_load_store(n));
326 effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
327 format %{ "weak CMPXCHGD $res, $mem_ptr, $src1, $src2; as bool; ptr" %}
328 ins_encode %{
329 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
330 __ cmpxchgd(CR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
331 MacroAssembler::MemBarNone,
332 MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
333 %}
334 ins_pipe(pipe_class_default);
335 %}
336
337 instruct compareAndExchangeN_regP_regN_regN_shenandoah(iRegNdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
338 match(Set res (CompareAndExchangeN mem_ptr (Binary src1 src2)));
339 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
340 !need_acquire_load_store(n));
341 effect(TEMP_DEF res, TEMP cr0);
342 format %{ "CMPXCHGW $res, $mem_ptr, $src1, $src2; as narrow oop" %}
343 ins_encode %{
344 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
345 __ cmpxchgw(CR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
346 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
347 noreg, nullptr, true);
348 %}
349 ins_pipe(pipe_class_default);
350 %}
351
352 instruct compareAndExchangeN_acq_regP_regN_regN_shenandoah(iRegNdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
353 match(Set res (CompareAndExchangeN mem_ptr (Binary src1 src2)));
354 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
355 need_acquire_load_store(n));
356 effect(TEMP_DEF res, TEMP cr0);
357 format %{ "CMPXCHGW $res, $mem_ptr, $src1, $src2; as narrow oop" %}
358 ins_encode %{
359 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
360 __ cmpxchgw(CR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
361 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
362 noreg, nullptr, true);
363
364 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
365 __ isync();
366 } else {
367 // isync would be sufficient in case of CompareAndExchangeAcquire, but we currently don't optimize for that.
368 __ sync();
369 }
370 %}
371 ins_pipe(pipe_class_default);
372 %}
373
374 instruct compareAndExchangeP_regP_regP_regP_shenandoah(iRegPdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
375 match(Set res (CompareAndExchangeP mem_ptr (Binary src1 src2)));
376 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
377 !need_acquire_load_store(n));
378 effect(TEMP_DEF res, TEMP cr0);
379 format %{ "CMPXCHGD $res, $mem_ptr, $src1, $src2; as ptr; ptr" %}
380 ins_encode %{
381 MemNode::MemOrd order = ((CompareAndSwapNode*)as_LoadStore())->order();
382 bool acquire = (order == MemNode::acquire) || (order == MemNode::seqcst);
383
384 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
385 __ cmpxchgd(CR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
386 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
387 noreg, nullptr, true);
388 %}
389 ins_pipe(pipe_class_default);
390 %}
391
392 instruct compareAndExchangeP_acq_regP_regP_regP_shenandoah(iRegPdst res, iRegPdst mem_ptr, iRegPsrc src1, iRegPsrc src2, flagsRegCR0 cr0) %{
393 match(Set res (CompareAndExchangeP mem_ptr (Binary src1 src2)));
394 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0) &&
395 need_acquire_load_store(n));
396 effect(TEMP_DEF res, TEMP cr0);
397 format %{ "CMPXCHGD $res, $mem_ptr, $src1, $src2; as ptr; ptr" %}
398 ins_encode %{
399 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
400 __ cmpxchgd(CR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
401 MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
402 noreg, nullptr, true);
403
404 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
405 __ isync();
406 } else {
407 // isync would be sufficient in case of CompareAndExchangeAcquire, but we currently don't optimize for that.
408 __ sync();
409 }
410 %}
411 ins_pipe(pipe_class_default);
412 %}
413
414 instruct getAndSetP_shenandoah(iRegPdst res, iRegPdst mem_ptr, iRegPsrc src, flagsRegCR0 cr0) %{
415 match(Set res (GetAndSetP mem_ptr src));
416 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0));
417 effect(TEMP_DEF res, TEMP cr0);
418 format %{ "GetAndSetP $res, $mem_ptr, $src" %}
419 ins_encode %{
420 __ getandsetd($res$$Register, $src$$Register, $mem_ptr$$Register,
421 MacroAssembler::cmpxchgx_hint_atomic_update());
422 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
423 __ isync();
424 } else {
425 __ sync();
426 }
427 %}
428 ins_pipe(pipe_class_default);
429 %}
430
431 instruct getAndSetN_shenandoah(iRegNdst res, iRegPdst mem_ptr, iRegNsrc src, flagsRegCR0 cr0) %{
432 match(Set res (GetAndSetN mem_ptr src));
433 predicate(UseShenandoahGC && (n->as_LoadStore()->barrier_data() != 0));
434 effect(TEMP_DEF res, TEMP cr0);
435 format %{ "GetAndSetN $res, $mem_ptr, $src" %}
436 ins_encode %{
437 __ getandsetw($res$$Register, $src$$Register, $mem_ptr$$Register,
438 MacroAssembler::cmpxchgx_hint_atomic_update());
439 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
440 __ isync();
441 } else {
442 __ sync();
443 }
444 %}
445 ins_pipe(pipe_class_default);
446 %}
447