1 //
2 // Copyright (c) 2018, Red Hat, Inc. All rights reserved.
3 // Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 source %{
27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp"
29 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
30 %}
31
32 instruct load_P_shenandoah(iRegPNoSp dst, memory mem)
33 %{
34 match(Set dst (LoadP mem));
35 predicate(UseShenandoahGC && n->as_Load()->barrier_data() != 0);
36 effect(TEMP_DEF dst);
37 // The main load is a candidate to implement implicit null checks.
38 ins_is_late_expanded_null_check_candidate(true);
39 format %{ "ld $dst, $mem\t# ptr, shenandoah late-barrier, #@loadP_shenandoah" %}
40 ins_encode %{
41 ShenandoahBarrierSet::assembler()->load_c2(this, masm, as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp), false);
42 %}
43 ins_cost(3 * LOAD_COST);
44 ins_pipe(iload_reg_mem);
45 %}
46
47 instruct load_N_shenandoah(iRegNNoSp dst, memory mem)
48 %{
49 match(Set dst (LoadN mem));
50 predicate(UseShenandoahGC && n->as_Load()->barrier_data() != 0);
51 effect(TEMP_DEF dst);
52 // The main load is a candidate to implement implicit null checks.
53 ins_is_late_expanded_null_check_candidate(true);
54 format %{ "lwu $dst, $mem\t# compressed ptr, shenandoah late-barrier, #@loadN_shenandoah" %}
55 ins_encode %{
56 ShenandoahBarrierSet::assembler()->load_c2(this, masm, as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp), true);
57 %}
58 ins_cost(3 * LOAD_COST);
59 ins_pipe(iload_reg_mem);
60 %}
61
62 instruct storeP_shenandoah(iRegP src, memory mem, iRegPNoSp tmp)
63 %{
64 match(Set mem (StoreP mem src));
65 predicate(UseShenandoahGC && n->as_Store()->barrier_data() != 0);
66 effect(TEMP tmp);
67 format %{ "sd $src, $mem\t# ptr, shenandoah late-barrier, #@storeP_shenandoah" %}
68 ins_encode %{
69 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
70 Address(as_Register($mem$$base), $mem$$disp), /* dst_narrow = */ false,
71 as_Register($src$$reg), /* src_narrow = */ false,
72 as_Register($tmp$$reg));
73 %}
74 ins_cost(3 * STORE_COST);
75 ins_pipe(istore_reg_mem);
76 %}
77
78 instruct storeN_shenandoah(iRegN src, memory mem, iRegPNoSp tmp)
79 %{
80 match(Set mem (StoreN mem src));
81 predicate(UseShenandoahGC && n->as_Store()->barrier_data() != 0);
82 effect(TEMP tmp);
83 format %{ "sw $src, $mem\t# compressed ptr, shenandoah late-barrier, #@storeN_shenandoah" %}
84 ins_encode %{
85 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
86 Address(as_Register($mem$$base), $mem$$disp), /* dst_narrow = */ true,
87 as_Register($src$$reg), /* src_narrow = */ true,
88 as_Register($tmp$$reg));
89 %}
90 ins_cost(3 * STORE_COST);
91 ins_pipe(istore_reg_mem);
92 %}
93
94 instruct encodePAndStoreN_shenandoah(iRegP src, memory mem, iRegPNoSp tmp)
95 %{
96 match(Set mem (StoreN mem (EncodeP src)));
97 predicate(UseShenandoahGC && n->as_Store()->barrier_data() != 0);
98 effect(TEMP tmp);
99 format %{
100 "encode_heap_oop $tmp, $src\n\t"
101 "sw $tmp, $mem\t# compressed ptr, shenandoah late-barrier, #@encodePAndStoreN_shenandoah"
102 %}
103 ins_encode %{
104 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
105 Address(as_Register($mem$$base), $mem$$disp), /* dst_narrow = */ true,
106 as_Register($src$$reg), /* src_narrow = */ false,
107 as_Register($tmp$$reg));
108 %}
109 ins_cost(4 * STORE_COST);
110 ins_pipe(istore_reg_mem);
111 %}
112
113 instruct compareAndSwap_P_A_shenandoah(iRegINoSp res, indirect mem, iRegPNoSp oldval, iRegP newval, iRegPNoSp tmp) %{
114 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
115 predicate(UseShenandoahGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
116 effect(TEMP_DEF res, TEMP tmp);
117 format %{ "cmpxchg_P_A_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval, #@compareAndSwap_P_shenandoah" %}
118 ins_encode %{
119 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
120 as_Register($res$$reg),
121 as_Register($mem$$base),
122 as_Register($oldval$$reg),
123 as_Register($newval$$reg),
124 as_Register($tmp$$reg),
125 /* exchange */ false,
126 /* is_narrow */ false,
127 /* acquire */ true);
128 %}
129 ins_cost(10 * DEFAULT_COST);
130 ins_pipe(pipe_slow);
131 %}
132
133 instruct compareAndSwap_P_shenandoah(iRegINoSp res, indirect mem, iRegPNoSp oldval, iRegP newval, iRegPNoSp tmp) %{
134 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
135 predicate(UseShenandoahGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
136 effect(TEMP_DEF res, TEMP tmp);
137 format %{ "cmpxchg_P_N_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval, #@compareAndSwap_P_shenandoah" %}
138 ins_encode %{
139 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
140 as_Register($res$$reg),
141 as_Register($mem$$base),
142 as_Register($oldval$$reg),
143 as_Register($newval$$reg),
144 as_Register($tmp$$reg),
145 /* exchange */ false,
146 /* is_narrow */ false,
147 /* acquire */ false);
148 %}
149 ins_cost(10 * DEFAULT_COST);
150 ins_pipe(pipe_slow);
151 %}
152
153 instruct compareAndSwap_N_A_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegN newval, iRegPNoSp tmp) %{
154 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
155 predicate(UseShenandoahGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
156 effect(TEMP_DEF res, TEMP tmp);
157 format %{ "cmpxchg_N_shenandoah $mem, $oldval, $newval\t# (narrow ptr) if $mem == $oldval then $mem <-- $newval, #@compareAndSwap_N_shenandoah" %}
158 ins_encode %{
159 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
160 as_Register($res$$reg),
161 as_Register($mem$$base),
162 as_Register($oldval$$reg),
163 as_Register($newval$$reg),
164 as_Register($tmp$$reg),
165 /* exchange */ false,
166 /* is_narrow */ true,
167 /* acquire */ true);
168 %}
169 ins_cost(10 * DEFAULT_COST);
170 ins_pipe(pipe_slow);
171 %}
172
173 instruct compareAndSwap_N_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegN newval, iRegPNoSp tmp) %{
174 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
175 predicate(UseShenandoahGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
176 effect(TEMP_DEF res, TEMP tmp);
177 format %{ "cmpxchg_N_shenandoah $mem, $oldval, $newval\t# (narrow ptr) if $mem == $oldval then $mem <-- $newval, #@compareAndSwap_N_shenandoah" %}
178 ins_encode %{
179 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
180 as_Register($res$$reg),
181 as_Register($mem$$base),
182 as_Register($oldval$$reg),
183 as_Register($newval$$reg),
184 as_Register($tmp$$reg),
185 /* exchange */ false,
186 /* is_narrow */ true,
187 /* acquire */ false);
188 %}
189 ins_cost(10 * DEFAULT_COST);
190 ins_pipe(pipe_slow);
191 %}
192
193 instruct compareAndExchange_N_A_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp) %{
194 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
195 predicate(UseShenandoahGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
196 effect(TEMP_DEF res, TEMP tmp);
197 format %{ "cae_N_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval, #@compareAndExchange_N_shenandoah" %}
198 ins_encode %{
199 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
200 as_Register($res$$reg),
201 as_Register($mem$$base),
202 as_Register($oldval$$reg),
203 as_Register($newval$$reg),
204 as_Register($tmp$$reg),
205 /* exchange */ true,
206 /* is_narrow */ true,
207 /* acquire */ true);
208 %}
209 ins_cost(10 * DEFAULT_COST);
210 ins_pipe(pipe_slow);
211 %}
212
213 instruct compareAndExchange_N_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp) %{
214 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
215 predicate(UseShenandoahGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
216 effect(TEMP_DEF res, TEMP tmp);
217 format %{ "cae_N_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval, #@compareAndExchange_N_shenandoah" %}
218 ins_encode %{
219 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
220 as_Register($res$$reg),
221 as_Register($mem$$base),
222 as_Register($oldval$$reg),
223 as_Register($newval$$reg),
224 as_Register($tmp$$reg),
225 /* exchange */ true,
226 /* is_narrow */ true,
227 /* acquire */ false);
228 %}
229 ins_cost(10 * DEFAULT_COST);
230 ins_pipe(pipe_slow);
231 %}
232
233 instruct compareAndExchange_P_A_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
234 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
235 predicate(UseShenandoahGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
236 effect(TEMP_DEF res, TEMP tmp);
237 format %{ "cae_P_shenandoah $res = $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval, #@compareAndExchange_P_shenandoah" %}
238 ins_encode %{
239 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
240 as_Register($res$$reg),
241 as_Register($mem$$base),
242 as_Register($oldval$$reg),
243 as_Register($newval$$reg),
244 as_Register($tmp$$reg),
245 /* exchange */ true,
246 /* is_narrow */ false,
247 /* acquire */ true);
248 %}
249 ins_cost(10 * DEFAULT_COST);
250 ins_pipe(pipe_slow);
251 %}
252
253 instruct compareAndExchange_P_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
254 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
255 predicate(UseShenandoahGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
256 effect(TEMP_DEF res, TEMP tmp);
257 format %{ "cae_P_shenandoah $res = $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval, #@compareAndExchange_P_shenandoah" %}
258 ins_encode %{
259 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
260 as_Register($res$$reg),
261 as_Register($mem$$base),
262 as_Register($oldval$$reg),
263 as_Register($newval$$reg),
264 as_Register($tmp$$reg),
265 /* exchange */ true,
266 /* is_narrow */ false,
267 /* acquire */ false);
268 %}
269 ins_cost(10 * DEFAULT_COST);
270 ins_pipe(pipe_slow);
271 %}
272
273 instruct weakCompareAndSwap_N_A_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegN newval, iRegPNoSp tmp) %{
274 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
275 predicate(UseShenandoahGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
276 effect(TEMP_DEF res, TEMP tmp);
277 format %{ "cae_N_weak_shenandoah $res = $mem, $oldval, $newval\t# (N, weak) if $mem == $oldval then $mem <-- $newval, #@weakCompareAndSwap_N_shenandoah" %}
278 ins_encode %{
279 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
280 as_Register($res$$reg),
281 as_Register($mem$$base),
282 as_Register($oldval$$reg),
283 as_Register($newval$$reg),
284 as_Register($tmp$$reg),
285 /* exchange */ false,
286 /* is_narrow */ true,
287 /* acquire */ true);
288 %}
289 ins_cost(10 * DEFAULT_COST);
290 ins_pipe(pipe_slow);
291 %}
292
293 instruct weakCompareAndSwap_N_shenandoah(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegN newval, iRegPNoSp tmp) %{
294 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
295 predicate(UseShenandoahGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
296 effect(TEMP_DEF res, TEMP tmp);
297 format %{ "cae_N_weak_shenandoah $res = $mem, $oldval, $newval\t# (N, weak) if $mem == $oldval then $mem <-- $newval, #@weakCompareAndSwap_N_shenandoah" %}
298 ins_encode %{
299 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
300 as_Register($res$$reg),
301 as_Register($mem$$base),
302 as_Register($oldval$$reg),
303 as_Register($newval$$reg),
304 as_Register($tmp$$reg),
305 /* exchange */ false,
306 /* is_narrow */ true,
307 /* acquire */ false);
308 %}
309 ins_cost(10 * DEFAULT_COST);
310 ins_pipe(pipe_slow);
311 %}
312
313 instruct weakCompareAndSwap_P_A_shenandoah(iRegINoSp res, indirect mem, iRegPNoSp oldval, iRegP newval, iRegPNoSp tmp) %{
314 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
315 predicate(UseShenandoahGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
316 effect(TEMP_DEF res, TEMP tmp);
317 format %{ "cae_P_weak_shenandoah $res = $mem, $oldval, $newval\t# (P, weak) if $mem == $oldval then $mem <-- $newval, #@weakCompareAndSwap_P_shenandoah" %}
318 ins_encode %{
319 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
320 as_Register($res$$reg),
321 as_Register($mem$$base),
322 as_Register($oldval$$reg),
323 as_Register($newval$$reg),
324 as_Register($tmp$$reg),
325 /* exchange */ false,
326 /* is_narrow */ false,
327 /* acquire */ true);
328 %}
329 ins_cost(10 * DEFAULT_COST);
330 ins_pipe(pipe_slow);
331 %}
332
333 instruct weakCompareAndSwap_P_shenandoah(iRegINoSp res, indirect mem, iRegPNoSp oldval, iRegP newval, iRegPNoSp tmp) %{
334 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
335 predicate(UseShenandoahGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
336 effect(TEMP_DEF res, TEMP tmp);
337 format %{ "cae_P_weak_shenandoah $res = $mem, $oldval, $newval\t# (P, weak) if $mem == $oldval then $mem <-- $newval, #@weakCompareAndSwap_P_shenandoah" %}
338 ins_encode %{
339 ShenandoahBarrierSet::assembler()->compare_and_set_c2(this, masm,
340 as_Register($res$$reg),
341 as_Register($mem$$base),
342 as_Register($oldval$$reg),
343 as_Register($newval$$reg),
344 as_Register($tmp$$reg),
345 /* exchange */ false,
346 /* is_narrow */ false,
347 /* acquire */ false);
348 %}
349 ins_cost(10 * DEFAULT_COST);
350 ins_pipe(pipe_slow);
351 %}
352
353 instruct getAndSet_P_shenandoah(indirect mem, iRegP newval, iRegPNoSp preval, iRegPNoSp tmp)
354 %{
355 match(Set preval (GetAndSetP mem newval));
356 predicate(UseShenandoahGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
357 effect(TEMP_DEF preval, TEMP tmp);
358 format %{ "get_and_set_P $preval, $newval, [$mem], #@getAndSet_P_shenandoah" %}
359 ins_encode %{
360 ShenandoahBarrierSet::assembler()->get_and_set_c2(this, masm,
361 as_Register($preval$$reg),
362 as_Register($newval$$reg),
363 as_Register($mem$$base),
364 as_Register($tmp$$reg),
365 /* acquire */ false);
366 %}
367 ins_cost(10 * DEFAULT_COST);
368 ins_pipe(pipe_slow);
369 %}
370
371 instruct getAndSet_P_A_shenandoah(indirect mem, iRegP newval, iRegPNoSp preval, iRegPNoSp tmp)
372 %{
373 match(Set preval (GetAndSetP mem newval));
374 predicate(UseShenandoahGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
375 effect(TEMP_DEF preval, TEMP tmp);
376 format %{ "get_and_set_P $preval, $newval, [$mem], #@getAndSet_P_shenandoah" %}
377 ins_encode %{
378 ShenandoahBarrierSet::assembler()->get_and_set_c2(this, masm,
379 as_Register($preval$$reg),
380 as_Register($newval$$reg),
381 as_Register($mem$$base),
382 as_Register($tmp$$reg),
383 /* acquire */ true);
384 %}
385 ins_cost(10 * DEFAULT_COST);
386 ins_pipe(pipe_slow);
387 %}
388
389 instruct getAndSet_N_A_shenandoah(indirect mem, iRegN newval, iRegNNoSp preval, iRegPNoSp tmp)
390 %{
391 match(Set preval (GetAndSetN mem newval));
392 predicate(UseShenandoahGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
393 effect(TEMP_DEF preval, TEMP tmp);
394 format %{ "get_and_set_N $preval, $newval, [$mem], #@getAndSet_N_shenandoah" %}
395 ins_encode %{
396 ShenandoahBarrierSet::assembler()->get_and_set_c2(this, masm,
397 as_Register($preval$$reg),
398 as_Register($newval$$reg),
399 as_Register($mem$$base),
400 as_Register($tmp$$reg),
401 /* acquire */ true);
402 %}
403 ins_cost(10 * DEFAULT_COST);
404 ins_pipe(pipe_slow);
405 %}
406
407 instruct getAndSet_N_shenandoah(indirect mem, iRegN newval, iRegNNoSp preval, iRegPNoSp tmp)
408 %{
409 match(Set preval (GetAndSetN mem newval));
410 predicate(UseShenandoahGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
411 effect(TEMP_DEF preval, TEMP tmp);
412 format %{ "get_and_set_N $preval, $newval, [$mem], #@getAndSet_N_shenandoah" %}
413 ins_encode %{
414 ShenandoahBarrierSet::assembler()->get_and_set_c2(this, masm,
415 as_Register($preval$$reg),
416 as_Register($newval$$reg),
417 as_Register($mem$$base),
418 as_Register($tmp$$reg),
419 /* acquire */ false);
420 %}
421 ins_cost(10 * DEFAULT_COST);
422 ins_pipe(pipe_slow);
423 %}