1 //
2 // Copyright (c) 2018, Red Hat, Inc. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 source %{
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 %}
30
31 instruct store_P_Normal_shenandoah(memory8 mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
32 %{
33 match(Set mem (StoreP mem src));
34 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
35 effect(TEMP tmp, KILL cr);
36 ins_cost(3*INSN_COST);
37 format %{ "str $src, $mem" %}
38 ins_encode %{
39 const Address addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
40 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
41 addr, /* dst_narrow = */ false,
42 $src$$Register, /* src_narrow = */ false,
43 $tmp$$Register);
44 %}
45 ins_pipe(pipe_class_memory);
46 %}
47
48 instruct store_P_Volatile_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
49 %{
50 match(Set mem (StoreP mem src));
51 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
52 effect(TEMP tmp, KILL cr);
53 ins_cost(VOLATILE_REF_COST);
54 format %{ "str $src, $mem" %}
55 ins_encode %{
56 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
57 Address($mem$$Register), /* dst_narrow = */ false,
58 $src$$Register, /* src_narrow = */ false,
59 $tmp$$Register);
60 %}
61 ins_pipe(pipe_class_memory);
62 %}
63
64 instruct store_N_Normal_shenandoah(memory4 mem, iRegN src, iRegNNoSp tmp, rFlagsReg cr)
65 %{
66 match(Set mem (StoreN mem src));
67 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
68 effect(TEMP tmp, KILL cr);
69 ins_cost(3*INSN_COST);
70 format %{ "str $src, $mem" %}
71 ins_encode %{
72 const Address addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
73 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
74 addr, /* dst_narrow = */ true,
75 $src$$Register, /* src_narrow = */ true,
76 $tmp$$Register);
77 %}
78 ins_pipe(pipe_class_memory);
79 %}
80
81 instruct store_N_Volatile_shenandoah(indirect mem, iRegN src, iRegNNoSp tmp, rFlagsReg cr)
82 %{
83 match(Set mem (StoreN mem src));
84 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
85 effect(TEMP tmp, KILL cr);
86 ins_cost(VOLATILE_REF_COST);
87 format %{ "str $src, $mem" %}
88 ins_encode %{
89 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
90 Address($mem$$Register), /* dst_narrow = */ true,
91 $src$$Register, /* src_narrow = */ true,
92 $tmp$$Register);
93 %}
94 ins_pipe(pipe_class_memory);
95 %}
96
97 instruct encodePAndStoreN_Normal_shenandoah(memory4 mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
98 %{
99 match(Set mem (StoreN mem (EncodeP src)));
100 predicate(UseShenandoahGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
101 effect(TEMP tmp, KILL cr);
102 ins_cost(4*INSN_COST);
103 format %{ "encode_heap_oop tmp, $src\n\t"
104 "str tmp, $mem\t# compressed ptr" %}
105 ins_encode %{
106 const Address addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
107 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
108 addr, /* dst_narrow = */ true,
109 $src$$Register, /* src_narrow = */ false,
110 $tmp$$Register);
111 %}
112 ins_pipe(pipe_class_memory);
113 %}
114
115 instruct encodePAndStoreN_Volatile_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
116 %{
117 match(Set mem (StoreN mem (EncodeP src)));
118 predicate(UseShenandoahGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
119 effect(TEMP tmp, KILL cr);
120 ins_cost(4*INSN_COST);
121 format %{ "encode_heap_oop tmp, $src\n\t"
122 "str tmp, $mem\t# compressed ptr" %}
123 ins_encode %{
124 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
125 Address($mem$$Register), /* dst_narrow = */ true,
126 $src$$Register, /* src_narrow = */ false,
127 $tmp$$Register);
128 %}
129 ins_pipe(pipe_class_memory);
130 %}
131
132 instruct compareAndSwap_P_shenandoah(iRegINoSp res, indirect mem, iRegPNoSp oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr)
133 %{
134 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
135 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
136 ins_cost(VOLATILE_REF_COST);
137 effect(TEMP_DEF res, TEMP tmp, KILL cr);
138 format %{
139 "cmpxchg_P_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
140 %}
141 ins_encode %{
142 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
143
144 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
145 $res$$Register,
146 $mem$$base$$Register,
147 $oldval$$Register,
148 $newval$$Register,
149 $tmp$$Register,
150 /* exchange */ false,
151 /* maybe_null */ true,
152 /* is_narrow */ false,
153 /* weak */ false);
154 %}
155
156 ins_pipe(pipe_slow);
157 %}
158
159 instruct compareAndSwap_N_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr)
160 %{
161 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
162 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
163 ins_cost(VOLATILE_REF_COST);
164 effect(TEMP_DEF res, TEMP tmp, KILL cr);
165 format %{
166 "cmpxchg_N_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
167 %}
168 ins_encode %{
169 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
170
171 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
172 $res$$Register,
173 $mem$$base$$Register,
174 $oldval$$Register,
175 $newval$$Register,
176 $tmp$$Register,
177 /* exchange */ false,
178 /* maybe_null */ true,
179 /* is_narrow */ true,
180 /* weak */ false);
181 %}
182
183 ins_pipe(pipe_slow);
184 %}
185
186 instruct compareAndExchange_N_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr)
187 %{
188 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
189 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
190 ins_cost(2*VOLATILE_REF_COST);
191 effect(TEMP_DEF res, TEMP tmp, KILL cr);
192 format %{
193 "cae_N_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
194 %}
195 ins_encode %{
196 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
197
198 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
199 $res$$Register,
200 $mem$$base$$Register,
201 $oldval$$Register,
202 $newval$$Register,
203 $tmp$$Register,
204 /* exchange */ true,
205 /* maybe_null */ (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull),
206 /* is_narrow */ true,
207 /* weak */ false);
208 %}
209 ins_pipe(pipe_slow);
210 %}
211
212 instruct compareAndExchange_P_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr)
213 %{
214 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
215 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
216 ins_cost(2*VOLATILE_REF_COST);
217 effect(TEMP_DEF res, TEMP tmp, KILL cr);
218 format %{
219 "cae_P_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
220 %}
221 ins_encode %{
222 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
223
224 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
225 $res$$Register,
226 $mem$$base$$Register,
227 $oldval$$Register,
228 $newval$$Register,
229 $tmp$$Register,
230 /* exchange */ true,
231 /* maybe_null */ (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull),
232 /* is_narrow */ false,
233 /* weak */ false);
234 %}
235 ins_pipe(pipe_slow);
236 %}
237
238 instruct weakCompareAndSwap_N_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr)
239 %{
240 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
241 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
242 ins_cost(VOLATILE_REF_COST);
243 effect(TEMP_DEF res, TEMP tmp, KILL cr);
244 format %{
245 "cae_N_weak_shenandoah $res = $mem, $oldval, $newval\t# (N, weak) if $mem == $oldval then $mem <-- $newval\n\t"
246 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
247 %}
248 ins_encode %{
249 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
250
251 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
252 $res$$Register,
253 $mem$$base$$Register,
254 $oldval$$Register,
255 $newval$$Register,
256 $tmp$$Register,
257 /* exchange */ false,
258 /* maybe_null */ true,
259 /* is_narrow */ true,
260 /* weak */ true);
261 %}
262 ins_pipe(pipe_slow);
263 %}
264
265 instruct weakCompareAndSwap_P_shenandoah(iRegINoSp res, indirect mem, iRegPNoSp oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr)
266 %{
267 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
268 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
269 ins_cost(VOLATILE_REF_COST);
270 effect(TEMP_DEF res, TEMP tmp, KILL cr);
271 format %{
272 "cae_P_weak_shenandoah $res = $mem, $oldval, $newval\t# (P, weak) if $mem == $oldval then $mem <-- $newval\n\t"
273 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
274 %}
275 ins_encode %{
276 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
277
278 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
279 $res$$Register,
280 $mem$$base$$Register,
281 $oldval$$Register,
282 $newval$$Register,
283 $tmp$$Register,
284 /* exchange */ false,
285 /* maybe_null */ true,
286 /* is_narrow */ false,
287 /* weak */ true);
288 %}
289 ins_pipe(pipe_slow);
290 %}
291
292 instruct getAndSet_P_shenandoah(indirect mem, iRegP newval, iRegPNoSp preval, iRegPNoSp tmp, rFlagsReg cr)
293 %{
294 match(Set preval (GetAndSetP mem newval));
295 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
296 effect(TEMP_DEF preval, TEMP tmp, KILL cr);
297 ins_cost(2*VOLATILE_REF_COST);
298 format %{ "get_and_set_P $preval, $newval, [$mem]" %}
299 ins_encode %{
300 ShenandoahBarrierSet::assembler()->get_and_set_c2(this, masm,
301 $preval$$Register,
302 $newval$$Register,
303 $mem$$Register,
304 $tmp$$Register);
305 %}
306 ins_pipe(pipe_serial);
307 %}
308
309 instruct getAndSet_N_shenandoah(indirect mem, iRegN newval, iRegNNoSp preval, iRegNNoSp tmp, rFlagsReg cr)
310 %{
311 match(Set preval (GetAndSetN mem newval));
312 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
313 effect(TEMP_DEF preval, TEMP tmp, KILL cr);
314 ins_cost(2*VOLATILE_REF_COST);
315 format %{ "get_and_set_N $preval, $newval, [$mem]" %}
316 ins_encode %{
317 ShenandoahBarrierSet::assembler()->get_and_set_c2(this, masm,
318 $preval$$Register,
319 $newval$$Register,
320 $mem$$Register,
321 $tmp$$Register);
322 %}
323 ins_pipe(pipe_serial);
324 %}
325
326 instruct load_P_Normal_shenandoah(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
327 %{
328 match(Set dst (LoadP mem));
329 predicate(UseShenandoahGC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
330 effect(TEMP_DEF dst, KILL cr);
331 // The main load is a candidate to implement implicit null checks.
332 ins_is_late_expanded_null_check_candidate(true);
333 format %{ "ldr $dst, $mem\t# ptr" %}
334 ins_encode %{
335 Address addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
336 if (addr.getMode() == Address::base_plus_offset) {
337 addr = __ legitimize_address(addr, /* size_in_memory */ 8, rscratch1);
338 }
339 ShenandoahBarrierSet::assembler()->load_c2(this, masm, $dst$$Register, addr);
340 %}
341 ins_cost(3*INSN_COST);
342 ins_pipe(pipe_class_memory);
343 %}
344
345 instruct load_P_Volatile_shenandoah(iRegPNoSp dst, indirect mem, rFlagsReg cr)
346 %{
347 match(Set dst (LoadP mem));
348 predicate(UseShenandoahGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
349 effect(TEMP_DEF dst, KILL cr);
350 // The main load is a candidate to implement implicit null checks.
351 ins_is_late_expanded_null_check_candidate(true);
352 format %{ "ldr $dst, $mem\t# ptr" %}
353 ins_encode %{
354 ShenandoahBarrierSet::assembler()->load_c2(this, masm, $dst$$Register, Address($mem$$Register));
355 %}
356 ins_cost(3*INSN_COST);
357 ins_pipe(pipe_class_memory);
358 %}
359
360 instruct load_N_Normal_shenandoah(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
361 %{
362 match(Set dst (LoadN mem));
363 predicate(UseShenandoahGC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
364 effect(TEMP_DEF dst, KILL cr);
365 // The main load is a candidate to implement implicit null checks.
366 ins_is_late_expanded_null_check_candidate(true);
367 format %{ "ldrw $dst, $mem\t# ptr" %}
368 ins_encode %{
369 Address addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
370 if (addr.getMode() == Address::base_plus_offset) {
371 addr = __ legitimize_address(addr, /* size_in_memory */ 4, rscratch1);
372 }
373 ShenandoahBarrierSet::assembler()->load_c2(this, masm, $dst$$Register, addr);
374 %}
375 ins_cost(3*INSN_COST);
376 ins_pipe(pipe_class_memory);
377 %}
378
379 instruct load_N_Volatile_shenandoah(iRegNNoSp dst, indirect mem, rFlagsReg cr)
380 %{
381 match(Set dst (LoadN mem));
382 predicate(UseShenandoahGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
383 effect(TEMP_DEF dst, KILL cr);
384 // The main load is a candidate to implement implicit null checks.
385 ins_is_late_expanded_null_check_candidate(true);
386 format %{ "ldrw $dst, $mem\t# ptr" %}
387 ins_encode %{
388 ShenandoahBarrierSet::assembler()->load_c2(this, masm, $dst$$Register, Address($mem$$Register));
389 %}
390 ins_cost(VOLATILE_REF_COST);
391 ins_pipe(pipe_class_memory);
392 %}
393