1 //
2 // Copyright (c) 2018, Red Hat, Inc. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 source %{
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 %}
30
31 instruct storeP_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
32 %{
33 match(Set mem (StoreP mem src));
34 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
35 effect(TEMP tmp, KILL cr);
36 ins_cost(INSN_COST);
37 format %{ "str $src, $mem\t# ptr" %}
38 ins_encode %{
39 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
40 $mem$$Register /* addr */,
41 $tmp$$Register /* pre_val */);
42 __ str($src$$Register, $mem$$Register);
43 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
44 $mem$$Register /* addr */,
45 $tmp$$Register /* tmp */);
46 %}
47 ins_pipe(istore_reg_mem);
48 %}
49
50 instruct storePVolatile_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
51 %{
52 match(Set mem (StoreP mem src));
53 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
54 effect(TEMP tmp, KILL cr);
55 ins_cost(VOLATILE_REF_COST);
56 format %{ "stlr $src, $mem\t# ptr" %}
57 ins_encode %{
58 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
59 $mem$$Register /* obj */,
60 $tmp$$Register /* pre_val */);
61 __ stlr($src$$Register, $mem$$Register);
62 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
63 $mem$$Register /* addr */,
64 $tmp$$Register /* tmp */);
65 %}
66 ins_pipe(pipe_class_memory);
67 %}
68
69 instruct storeN_shenandoah(indirect mem, iRegN src, iRegPNoSp tmp, rFlagsReg cr)
70 %{
71 match(Set mem (StoreN mem src));
72 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
73 effect(TEMP tmp, KILL cr);
74 ins_cost(INSN_COST);
75 format %{ "strw $src, $mem\t# compressed ptr" %}
76 ins_encode %{
77 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
78 $mem$$Register /* obj */,
79 $tmp$$Register /* pre_val */);
80 __ strw($src$$Register, $mem$$Register);
81 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
82 $mem$$Register /* addr */,
83 $tmp$$Register /* tmp */);
84 %}
85 ins_pipe(istore_reg_mem);
86 %}
87
88 instruct storeNVolatile_shenandoah(indirect mem, iRegN src, iRegPNoSp tmp, rFlagsReg cr)
89 %{
90 match(Set mem (StoreN mem src));
91 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
92 effect(TEMP tmp, KILL cr);
93 ins_cost(VOLATILE_REF_COST);
94 format %{ "stlrw $src, $mem\t# compressed ptr" %}
95 ins_encode %{
96 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
97 $mem$$Register /* obj */,
98 $tmp$$Register /* pre_val */);
99 __ stlrw($src$$Register, $mem$$Register);
100 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
101 $mem$$Register /* addr */,
102 $tmp$$Register /* tmp */);
103 %}
104 ins_pipe(pipe_class_memory);
105 %}
106
107 instruct encodePAndStoreN_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
108 %{
109 match(Set mem (StoreN mem (EncodeP src)));
110 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
111 effect(TEMP tmp, KILL cr);
112 ins_cost(INSN_COST);
113 format %{ "encode_heap_oop $tmp, $src\n\t"
114 "strw $tmp, $mem\t# compressed ptr" %}
115 ins_encode %{
116 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
117 $mem$$Register /* obj */,
118 $tmp$$Register /* pre_val */);
119 if ((barrier_data() & ShenandoahBarrierCardMarkNotNull) == 0) {
120 __ encode_heap_oop($tmp$$Register, $src$$Register);
121 } else {
122 __ encode_heap_oop_not_null($tmp$$Register, $src$$Register);
123 }
124 __ strw($tmp$$Register, $mem$$Register);
125 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
126 $mem$$Register /* addr */,
127 $tmp$$Register /* tmp */);
128 %}
129 ins_pipe(istore_reg_mem);
130 %}
131
132 instruct encodePAndStoreNVolatile_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
133 %{
134 match(Set mem (StoreN mem (EncodeP src)));
135 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
136 effect(TEMP tmp, KILL cr);
137 ins_cost(VOLATILE_REF_COST);
138 format %{ "encode_heap_oop $tmp, $src\n\t"
139 "stlrw $tmp, $mem\t# compressed ptr" %}
140 ins_encode %{
141 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
142 $mem$$Register /* obj */,
143 $tmp$$Register /* pre_val */);
144 if ((barrier_data() & ShenandoahBarrierCardMarkNotNull) == 0) {
145 __ encode_heap_oop($tmp$$Register, $src$$Register);
146 } else {
147 __ encode_heap_oop_not_null($tmp$$Register, $src$$Register);
148 }
149 __ stlrw($tmp$$Register, $mem$$Register);
150 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
151 $mem$$Register /* addr */,
152 $tmp$$Register /* tmp */);
153 %}
154 ins_pipe(pipe_class_memory);
155 %}
156
157 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
158
159 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
160 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
161 ins_cost(2 * VOLATILE_REF_COST);
162
163 effect(TEMP_DEF res, TEMP tmp, KILL cr);
164
165 format %{
166 "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
167 %}
168
169 ins_encode %{
170 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
171 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
172 noreg /* obj */,
173 $oldval$$Register /* pre_val */);
174 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
175 /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false);
176 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
177 $mem$$Register /* addr */,
178 $tmp$$Register /* tmp */);
179 %}
180
181 ins_pipe(pipe_slow);
182 %}
183
184 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp, rFlagsReg cr) %{
185
186 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
187 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
188 ins_cost(2 * VOLATILE_REF_COST);
189
190 effect(TEMP_DEF res, TEMP tmp, KILL cr);
191
192 format %{
193 "cmpxchgw_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
194 %}
195
196 ins_encode %{
197 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
198 guarantee(UseCompressedOops, "must be compressed oops");
199 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
200 __ decode_heap_oop($tmp$$Register, $oldval$$Register);
201 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
202 noreg /* obj */,
203 $tmp$$Register /* pre_val */);
204 }
205 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
206 /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false);
207 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
208 $mem$$Register /* addr */,
209 $tmp$$Register /* tmp */);
210 %}
211
212 ins_pipe(pipe_slow);
213 %}
214
215 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegINoSp tmp, rFlagsReg cr) %{
216
217 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
218 predicate(needs_acquiring_load_exclusive(n));
219 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
220 ins_cost(VOLATILE_REF_COST);
221
222 effect(TEMP_DEF res, TEMP tmp, KILL cr);
223
224 format %{
225 "cmpxchg_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
226 %}
227
228 ins_encode %{
229 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
230 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
231 noreg /* obj */,
232 $oldval$$Register /* pre_val */);
233 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
234 /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false);
235 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
236 $mem$$Register /* addr */,
237 $tmp$$Register /* tmp */);
238 %}
239
240 ins_pipe(pipe_slow);
241 %}
242
243 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp, rFlagsReg cr) %{
244
245 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
246 predicate(needs_acquiring_load_exclusive(n));
247 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
248 ins_cost(VOLATILE_REF_COST);
249
250 effect(TEMP_DEF res, TEMP tmp, KILL cr);
251
252 format %{
253 "cmpxchgw_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
254 %}
255
256 ins_encode %{
257 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
258 guarantee(UseCompressedOops, "must be compressed oops");
259 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
260 __ decode_heap_oop($tmp$$Register, $oldval$$Register);
261 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
262 noreg /* obj */,
263 $tmp$$Register /* pre_val */);
264 }
265 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
266 /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false);
267 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
268 $mem$$Register /* addr */,
269 $tmp$$Register /* tmp */);
270 %}
271
272 ins_pipe(pipe_slow);
273 %}
274
275 instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp, rFlagsReg cr) %{
276 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
277 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
278 ins_cost(2 * VOLATILE_REF_COST);
279 effect(TEMP_DEF res, TEMP tmp, KILL cr);
280 format %{
281 "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
282 %}
283 ins_encode %{
284 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
285 guarantee(UseCompressedOops, "must be compressed oops");
286 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
287 __ decode_heap_oop($tmp$$Register, $oldval$$Register);
288 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
289 noreg /* obj */,
290 $tmp$$Register /* pre_val */);
291 }
292 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
293 /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true);
294 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
295 $res$$Register /* obj */,
296 $mem$$Register /* addr */,
297 $tmp$$Register /* tmp */,
298 true /* narrow */,
299 true /* maybe_null */);
300 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
301 $mem$$Register /* addr */,
302 $tmp$$Register /* tmp */);
303 %}
304 ins_pipe(pipe_slow);
305 %}
306
307 instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
308 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
309 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
310 ins_cost(2 * VOLATILE_REF_COST);
311 effect(TEMP_DEF res, TEMP tmp, KILL cr);
312 format %{
313 "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
314 %}
315 ins_encode %{
316 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
317 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
318 noreg /* obj */,
319 $oldval$$Register /* pre_val */);
320 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
321 /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true);
322 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
323 $res$$Register /* obj */,
324 $mem$$Register /* addr */,
325 $tmp$$Register /* tmp */,
326 false /* narrow */,
327 true /* maybe_null */);
328 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
329 $mem$$Register /* addr */,
330 $tmp$$Register /* tmp */);
331 %}
332 ins_pipe(pipe_slow);
333 %}
334
335 instruct compareAndExchangeNAcq_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp, rFlagsReg cr) %{
336 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
337 predicate(needs_acquiring_load_exclusive(n));
338 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
339 ins_cost(VOLATILE_REF_COST);
340 effect(TEMP_DEF res, TEMP tmp, KILL cr);
341 format %{
342 "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
343 %}
344 ins_encode %{
345 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
346 guarantee(UseCompressedOops, "must be compressed oops");
347 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
348 __ decode_heap_oop($tmp$$Register, $oldval$$Register);
349 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
350 noreg /* obj */,
351 $tmp$$Register /* pre_val */);
352 }
353 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
354 /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ true);
355 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
356 $res$$Register /* obj */,
357 $mem$$Register /* addr */,
358 $tmp$$Register /* tmp */,
359 true /* narrow */,
360 true /* maybe_null */);
361 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
362 $mem$$Register /* addr */,
363 $tmp$$Register /* tmp */);
364 %}
365 ins_pipe(pipe_slow);
366 %}
367
368 instruct compareAndExchangePAcq_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
369 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
370 predicate(needs_acquiring_load_exclusive(n));
371 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
372 ins_cost(VOLATILE_REF_COST);
373 effect(TEMP_DEF res, TEMP tmp, KILL cr);
374 format %{
375 "cmpxchg_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
376 %}
377 ins_encode %{
378 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
379 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
380 noreg /* obj */,
381 $oldval$$Register /* pre_val */);
382 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
383 /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ true);
384 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
385 $res$$Register /* obj */,
386 $mem$$Register /* addr */,
387 $tmp$$Register /* tmp */,
388 false /* narrow */,
389 true /* maybe_null */);
390 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
391 $mem$$Register /* addr */,
392 $tmp$$Register /* tmp */);
393 %}
394 ins_pipe(pipe_slow);
395 %}
396
397 instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp, rFlagsReg cr) %{
398 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
399 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
400 ins_cost(2 * VOLATILE_REF_COST);
401 effect(TEMP_DEF res, TEMP tmp, KILL cr);
402 format %{
403 "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
404 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
405 %}
406 ins_encode %{
407 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
408 guarantee(UseCompressedOops, "must be compressed oops");
409 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
410 __ decode_heap_oop($tmp$$Register, $oldval$$Register);
411 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
412 noreg /* obj */,
413 $tmp$$Register /* pre_val */);
414 }
415 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
416 /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false);
417 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
418 $mem$$Register /* addr */,
419 $tmp$$Register /* tmp */);
420 %}
421 ins_pipe(pipe_slow);
422 %}
423
424 instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegINoSp tmp, rFlagsReg cr) %{
425 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
426 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
427 ins_cost(2 * VOLATILE_REF_COST);
428 effect(TEMP_DEF res, TEMP tmp, KILL cr);
429 format %{
430 "cmpxchg_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
431 %}
432 ins_encode %{
433 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
434 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
435 noreg /* obj */,
436 $oldval$$Register /* pre_val */);
437 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
438 /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false);
439 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
440 $mem$$Register /* addr */,
441 $tmp$$Register /* tmp */);
442 %}
443 ins_pipe(pipe_slow);
444 %}
445
446 instruct weakCompareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp, rFlagsReg cr) %{
447 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
448 predicate(needs_acquiring_load_exclusive(n));
449 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
450 ins_cost(VOLATILE_REF_COST);
451 effect(TEMP_DEF res, TEMP tmp, KILL cr);
452 format %{
453 "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
454 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
455 %}
456 ins_encode %{
457 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
458 guarantee(UseCompressedOops, "must be compressed oops");
459 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
460 __ decode_heap_oop($tmp$$Register, $oldval$$Register);
461 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
462 noreg /* obj */,
463 $tmp$$Register /* pre_val */);
464 }
465 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
466 /*acquire*/ true, /*release*/ true, /*weak*/ true, /*is_cae*/ false);
467 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
468 $mem$$Register /* addr */,
469 $tmp$$Register /* tmp */);
470 %}
471 ins_pipe(pipe_slow);
472 %}
473
474 instruct weakCompareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
475 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
476 predicate(needs_acquiring_load_exclusive(n));
477 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
478 ins_cost(VOLATILE_REF_COST);
479 effect(TEMP_DEF res, TEMP tmp, KILL cr);
480 format %{
481 "cmpxchg_acq_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
482 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
483 %}
484 ins_encode %{
485 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
486 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
487 noreg /* obj */,
488 $oldval$$Register /* pre_val */);
489 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm, $mem$$base$$Register, $oldval$$Register, $newval$$Register, $res$$Register,
490 /*acquire*/ true, /*release*/ true, /*weak*/ true, /*is_cae*/ false);
491 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
492 $mem$$Register /* addr */,
493 $tmp$$Register /* tmp */);
494 %}
495 ins_pipe(pipe_slow);
496 %}
497
498 instruct getAndSetP_shenandoah(indirect mem, iRegP newval, iRegPNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
499 %{
500 match(Set preval (GetAndSetP mem newval));
501 predicate(UseShenandoahGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
502 effect(TEMP_DEF preval, TEMP tmp, KILL cr);
503 ins_cost(2 * VOLATILE_REF_COST);
504 format %{ "atomic_xchg $preval, $newval, [$mem]" %}
505 ins_encode %{
506 assert_different_registers($mem$$Register, $newval$$Register);
507 __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
508 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
509 noreg /* obj */,
510 $preval$$Register /* pre_val */);
511 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
512 $preval$$Register /* obj */,
513 $mem$$Register /* addr */,
514 $tmp$$Register /* tmp */,
515 false /* narrow */,
516 true /* maybe_null */);
517 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
518 $mem$$Register /* addr */,
519 $tmp$$Register /* tmp */);
520 %}
521 ins_pipe(pipe_serial);
522 %}
523
524 instruct getAndSetPAcq_shenandoah(indirect mem, iRegP newval, iRegPNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
525 %{
526 match(Set preval (GetAndSetP mem newval));
527 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
528 effect(TEMP_DEF preval, TEMP tmp, KILL cr);
529 ins_cost(VOLATILE_REF_COST);
530 format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %}
531 ins_encode %{
532 assert_different_registers($mem$$Register, $newval$$Register);
533 __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
534 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
535 noreg /* obj */,
536 $preval$$Register /* pre_val */);
537 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
538 $preval$$Register /* obj */,
539 $mem$$Register /* addr */,
540 $tmp$$Register /* tmp */,
541 false /* narrow */,
542 true /* maybe_null */);
543 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
544 $mem$$Register /* addr */,
545 $tmp$$Register /* tmp */);
546 %}
547 ins_pipe(pipe_serial);
548 %}
549
550 instruct getAndSetN_shenandoah(indirect mem, iRegN newval, iRegNNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
551 %{
552 match(Set preval (GetAndSetN mem newval));
553 predicate(UseShenandoahGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
554 effect(TEMP preval, TEMP tmp, KILL cr);
555 ins_cost(2 * VOLATILE_REF_COST);
556 format %{ "atomic_xchgw $preval, $newval, [$mem]" %}
557 ins_encode %{
558 assert_different_registers($mem$$Register, $newval$$Register);
559 __ atomic_xchgw($preval$$Register, $newval$$Register, $mem$$Register);
560 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
561 __ decode_heap_oop($tmp$$Register, $preval$$Register);
562 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
563 noreg /* obj */,
564 $tmp$$Register /* pre_val */);
565 }
566 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
567 $preval$$Register /* obj */,
568 $mem$$Register /* addr */,
569 $tmp$$Register /* tmp */,
570 true /* narrow */,
571 true /* maybe_null */);
572 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
573 $mem$$Register /* addr */,
574 $tmp$$Register /* tmp */);
575 %}
576 ins_pipe(pipe_serial);
577 %}
578
579 instruct getAndSetNAcq_shenandoah(indirect mem, iRegN newval, iRegNNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
580 %{
581 match(Set preval (GetAndSetN mem newval));
582 predicate(UseShenandoahGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
583 effect(TEMP preval, TEMP tmp, KILL cr);
584 ins_cost(VOLATILE_REF_COST);
585 format %{ "atomic_xchgw_acq $preval, $newval, [$mem]" %}
586 ins_encode %{
587 assert_different_registers($mem$$Register, $newval$$Register);
588 __ atomic_xchgalw($preval$$Register, $newval$$Register, $mem$$Register);
589 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
590 __ decode_heap_oop($tmp$$Register, $preval$$Register);
591 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
592 noreg /* obj */,
593 $tmp$$Register /* pre_val */);
594 }
595 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
596 $preval$$Register /* obj */,
597 $mem$$Register /* addr */,
598 $tmp$$Register /* tmp */,
599 true /* narrow */,
600 true /* maybe_null */);
601 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
602 $mem$$Register /* addr */,
603 $tmp$$Register /* tmp */);
604 %}
605 ins_pipe(pipe_serial);
606 %}
607
608 instruct loadP_shenandoah(iRegPNoSp dst, indirect mem, rFlagsReg cr)
609 %{
610 // This instruction does not need an acquiring counterpart because it is only
611 // used for reference loading (Reference::get()).
612 match(Set dst (LoadP mem));
613 predicate(UseShenandoahGC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
614 effect(TEMP_DEF dst, KILL cr);
615 ins_cost(4 * INSN_COST);
616 format %{ "ldr $dst, $mem\t# ptr" %}
617 ins_encode %{
618 __ ldr($dst$$Register, $mem$$Register);
619 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
620 noreg /* obj */,
621 $dst$$Register /* pre_val */);
622 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
623 $dst$$Register /* obj */,
624 $mem$$Register /* addr */,
625 noreg /* noreg - not needed */,
626 false /* narrow */,
627 true /* maybe_null */);
628 %}
629 ins_pipe(iload_reg_mem);
630 %}
631
632 instruct loadP_volatile_shenandoah(iRegPNoSp dst, indirect mem, rFlagsReg cr)
633 %{
634 // This instruction does not need an acquiring counterpart because it is only
635 // used for reference loading (Reference::get()).
636 match(Set dst (LoadP mem));
637 predicate(UseShenandoahGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
638 effect(TEMP dst, KILL cr);
639 ins_cost(4 * INSN_COST);
640 format %{ "ldar $dst, $mem\t# ptr" %}
641 ins_encode %{
642 __ ldar($dst$$Register, $mem$$Register);
643 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
644 noreg /* obj */,
645 $dst$$Register /* pre_val */);
646 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
647 $dst$$Register /* obj */,
648 $mem$$Register /* addr */,
649 noreg /* noreg - not needed */,
650 false /* narrow */,
651 true /* maybe_null */);
652 %}
653 ins_pipe(iload_reg_mem);
654 %}
655
656 instruct loadN_shenandoah(iRegNNoSp dst, indirect mem, iRegPNoSp tmp, rFlagsReg cr) %{
657 match(Set dst (LoadN mem));
658 predicate(UseShenandoahGC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
659 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
660 ins_cost(INSN_COST * 4);
661 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
662 ins_encode %{
663 __ ldrw($dst$$Register, $mem$$Register);
664 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
665 __ decode_heap_oop($tmp$$Register, $dst$$Register);
666 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
667 noreg /* obj */,
668 $tmp$$Register /* pre_val */);
669 }
670 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
671 $dst$$Register /* obj */,
672 $mem$$Register /* addr */,
673 $tmp$$Register /* tmp */,
674 true /* narrow */,
675 true /* maybe_null */);
676 %}
677 ins_pipe(ialu_reg);
678 %}
679
680 instruct loadN_volatile_shenandoah(iRegNNoSp dst, indirect mem, iRegPNoSp tmp, rFlagsReg cr) %{
681 predicate(UseShenandoahGC && /*needs_acquiring_load(n) && */ n->as_Load()->barrier_data() != 0);
682 match(Set dst (LoadN mem));
683 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
684 ins_cost(INSN_COST * 3);
685 format %{ "ldarw $dst, $mem\t# ptr"
686 "decode_heap_oop $dst, $dst"
687 %}
688 ins_encode %{
689 __ ldarw($dst$$Register, $mem$$Register);
690 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
691 __ decode_heap_oop($tmp$$Register, $dst$$Register);
692 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
693 noreg /* obj */,
694 $tmp$$Register /* pre_val */);
695 }
696 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
697 $dst$$Register /* obj */,
698 $mem$$Register /* addr */,
699 $tmp$$Register /* tmp */,
700 true /* narrow */,
701 true /* maybe_null */);
702 %}
703 ins_pipe(ialu_reg);
704 %}