1 //
2 // Copyright (c) 2018, Red Hat, Inc. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 source %{
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 %}
30
31 // ---------------------------------- LOADS ---------------------------------------
32 //
33
34 instruct loadP_shenandoah(rRegP dst, memory mem, rRegP tmp, rFlagsReg cr)
35 %{
36 match(Set dst (LoadP mem));
37 predicate(UseShenandoahGC && n->as_Load()->barrier_data() != 0);
38 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
39 // The main load is a candidate to implement implicit null checks.
40 ins_is_late_expanded_null_check_candidate(true);
41 format %{ "movq $dst, $mem\t# ptr" %}
42 ins_encode %{
43 ShenandoahBarrierSet::assembler()->load_c2(this, masm,
44 $dst$$Register,
45 $mem$$Address,
46 /* narrow = */ false,
47 $tmp$$Register
48 );
49 %}
50 ins_cost(125);
51 ins_pipe(ialu_reg_mem);
52 %}
53
54 instruct loadN_shenandoah(rRegN dst, memory mem, rRegP tmp, rFlagsReg cr)
55 %{
56 match(Set dst (LoadN mem));
57 predicate(UseShenandoahGC && n->as_Load()->barrier_data() != 0);
58 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
59 // The main load is a candidate to implement implicit null checks.
60 ins_is_late_expanded_null_check_candidate(true);
61 format %{ "movl $dst, $mem\t# compressed ptr" %}
62 ins_encode %{
63 ShenandoahBarrierSet::assembler()->load_c2(this, masm,
64 $dst$$Register,
65 $mem$$Address,
66 /* narrow = */ true,
67 $tmp$$Register
68 );
69 %}
70 ins_cost(125);
71 ins_pipe(ialu_reg_mem);
72 %}
73
74 // ---------------------------- LOAD SPECIALIZATIONS ------------------------
75 // These rules match x86.ad match rules with barrier_data() == 0.
76 // For these cases, Shenandoah does not care about applying the GC barriers:
77 // this applies to everywhere we do not expose the untreated oop to the rest
78 // of the system.
79 //
80
81 instruct testP_mem_shenandoah(rFlagsReg cr, memory op, immP0 zero)
82 %{
83 predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
84 UseShenandoahGC && n->in(1)->as_Load()->barrier_data() != 0);
85 match(Set cr (CmpP (LoadP op) zero));
86
87 ins_cost(500); // XXX
88 format %{ "testq $op, 0xffffffffffffffff\t# ptr" %}
89 ins_encode %{
90 __ testq($op$$Address, 0xFFFFFFFF);
91 %}
92 ins_pipe(ialu_cr_reg_imm);
93 %}
94
95 instruct testP_mem_reg0_shenandoah(rFlagsReg cr, memory mem, immP0 zero)
96 %{
97 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
98 UseShenandoahGC && n->in(1)->as_Load()->barrier_data() != 0);
99 match(Set cr (CmpP (LoadP mem) zero));
100
101 format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %}
102 ins_encode %{
103 __ cmpq(r12, $mem$$Address);
104 %}
105 ins_pipe(ialu_cr_reg_mem);
106 %}
107
108 instruct compN_rReg_mem_shenandoah(rFlagsRegU cr, rRegN src, memory mem)
109 %{
110 predicate(UseShenandoahGC && n->in(2)->as_Load()->barrier_data() != 0);
111 match(Set cr (CmpN src (LoadN mem)));
112
113 format %{ "cmpl $src, $mem\t# compressed ptr" %}
114 ins_encode %{
115 __ cmpl($src$$Register, $mem$$Address);
116 %}
117 ins_pipe(ialu_cr_reg_mem);
118 %}
119
120 instruct testN_mem_shenandoah(rFlagsReg cr, memory mem, immN0 zero)
121 %{
122 predicate(CompressedOops::base() != nullptr &&
123 UseShenandoahGC && n->in(1)->as_Load()->barrier_data() != 0);
124 match(Set cr (CmpN (LoadN mem) zero));
125
126 ins_cost(500); // XXX
127 format %{ "testl $mem, 0xffffffff\t# compressed ptr" %}
128 ins_encode %{
129 __ cmpl($mem$$Address, (int)0xFFFFFFFF);
130 %}
131 ins_pipe(ialu_cr_reg_mem);
132 %}
133
134 instruct testN_mem_reg0_shenandoah(rFlagsReg cr, memory mem, immN0 zero)
135 %{
136 predicate(CompressedOops::base() == nullptr &&
137 UseShenandoahGC && n->in(1)->as_Load()->barrier_data() != 0);
138 match(Set cr (CmpN (LoadN mem) zero));
139
140 format %{ "cmpl R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
141 ins_encode %{
142 __ cmpl(r12, $mem$$Address);
143 %}
144 ins_pipe(ialu_cr_reg_mem);
145 %}
146
147 // ---------------------------------- STORES ---------------------------------------
148 //
149
150 instruct storeP_shenandoah(memory mem, any_RegP src, rRegP tmp, rFlagsReg cr)
151 %{
152 match(Set mem (StoreP mem src));
153 predicate(UseShenandoahGC && n->as_Store()->barrier_data() != 0);
154 effect(TEMP tmp, KILL cr);
155 ins_cost(125); // XXX
156 format %{ "movq $mem, $src\t# ptr" %}
157 ins_encode %{
158 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
159 $mem$$Address, /* dst_narrow = */ false,
160 $src$$Register, /* src_narrow = */ false,
161 $tmp$$Register);
162 %}
163 ins_pipe(ialu_mem_reg);
164 %}
165
166 instruct storeN_shenandoah(memory mem, rRegN src, rRegP tmp, rFlagsReg cr)
167 %{
168 match(Set mem (StoreN mem src));
169 predicate(UseShenandoahGC && n->as_Store()->barrier_data() != 0);
170 effect(TEMP tmp, KILL cr);
171 ins_cost(125); // XXX
172 format %{ "movl $mem, $src\t# ptr" %}
173 ins_encode %{
174 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
175 $mem$$Address, /* dst_narrow = */ true,
176 $src$$Register, /* src_narrow = */ true,
177 $tmp$$Register);
178 %}
179 ins_pipe(ialu_mem_reg);
180 %}
181
182 instruct encodePAndStoreN_shenandoah(memory mem, any_RegP src, rRegP tmp, rFlagsReg cr)
183 %{
184 match(Set mem (StoreN mem (EncodeP src)));
185 predicate(UseShenandoahGC && n->as_Store()->barrier_data() != 0);
186 effect(TEMP tmp, KILL cr);
187 ins_cost(125); // XXX
188 format %{ "encode_heap_oop $src\n\t"
189 "movl $mem, $src\t# ptr" %}
190 ins_encode %{
191 ShenandoahBarrierSet::assembler()->store_c2(this, masm,
192 $mem$$Address, /* dst_narrow = */ true,
193 $src$$Register, /* src_narrow = */ false,
194 $tmp$$Register);
195 %}
196 ins_pipe(ialu_mem_reg);
197 %}
198
199 // ---------------------------------- LOAD-STORES ---------------------------------------
200 //
201
202 instruct compareAndSwapP_shenandoah(rRegI res,
203 indirect mem,
204 rRegP tmp1, rRegP tmp2,
205 rax_RegP oldval, rRegP newval,
206 rFlagsReg cr)
207 %{
208 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
209 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
210 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
211 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
212
213 format %{ "shenandoah_cas_oop $mem,$newval" %}
214
215 ins_encode %{
216 guarantee(!UseCompressedOops, "must not be compressed oops");
217 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
218 __ movptr($tmp1$$Register, $oldval$$Register);
219 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
220 noreg /* addr */,
221 $tmp1$$Register /* pre_val */,
222 $tmp2$$Register /* tmp */);
223 }
224 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
225 $res$$Register, $mem$$Address, $oldval$$Register, $newval$$Register, $tmp1$$Register, $tmp2$$Register,
226 /*exchange*/ false);
227 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
228 $mem$$Register /* addr */,
229 $tmp1$$Register /* addr_tmp */,
230 $tmp2$$Register /* tmp */);
231 %}
232 ins_pipe( pipe_cmpxchg );
233 %}
234
235 instruct compareAndSwapN_shenandoah(rRegI res,
236 indirect mem,
237 rRegP tmp1, rRegP tmp2,
238 rax_RegN oldval, rRegN newval,
239 rFlagsReg cr) %{
240 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
241 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
242 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
243 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
244
245 format %{ "shenandoah_cas_oop $mem,$newval" %}
246
247 ins_encode %{
248 guarantee(UseCompressedOops, "must be compressed oops");
249 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
250 __ movl($tmp1$$Register, $oldval$$Register);
251 __ decode_heap_oop($tmp1$$Register);
252 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
253 noreg /* addr */,
254 $tmp1$$Register /* pre_val */,
255 $tmp2$$Register /* tmp */);
256 }
257 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
258 $res$$Register, $mem$$Address, $oldval$$Register, $newval$$Register, $tmp1$$Register, $tmp2$$Register,
259 /*exchange*/ false);
260 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
261 $mem$$Register /* addr */,
262 $tmp1$$Register /* addr_tmp */,
263 $tmp2$$Register /* tmp */);
264 %}
265 ins_pipe( pipe_cmpxchg );
266 %}
267
268 instruct compareAndExchangeN_shenandoah(indirect mem,
269 rax_RegN oldval, rRegN newval,
270 rRegP tmp1, rRegP tmp2, rRegP tmp3,
271 rFlagsReg cr) %{
272 match(Set oldval (CompareAndExchangeN mem (Binary oldval newval)));
273 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
274 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
275
276 format %{ "shenandoah_cas_oop $mem,$newval" %}
277
278 ins_encode %{
279 guarantee(UseCompressedOops, "must be compressed oops");
280 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
281 __ movl($tmp1$$Register, $oldval$$Register);
282 __ decode_heap_oop($tmp1$$Register);
283 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
284 noreg /* addr */,
285 $tmp1$$Register /* pre_val */,
286 $tmp2$$Register /* tmp */);
287 }
288 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
289 noreg, $mem$$Address, $oldval$$Register, $newval$$Register, $tmp1$$Register, $tmp2$$Register,
290 /*exchange*/ true);
291 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
292 $oldval$$Register /* obj */,
293 $mem$$Register /* addr */,
294 $tmp1$$Register /* tmp1 */,
295 $tmp2$$Register /* tmp2 */,
296 $tmp3$$Register /* tmp3 */,
297 true /* narrow */);
298 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
299 $mem$$Register /* addr */,
300 $tmp1$$Register /* addr_tmp */,
301 $tmp2$$Register /* tmp */);
302 %}
303 ins_pipe( pipe_cmpxchg );
304 %}
305
306 instruct compareAndExchangeP_shenandoah(indirect mem,
307 rax_RegP oldval, rRegP newval,
308 rRegP tmp1, rRegP tmp2,
309 rFlagsReg cr)
310 %{
311 match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
312 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
313 effect(KILL cr, TEMP tmp1, TEMP tmp2);
314 ins_cost(1000);
315
316 format %{ "shenandoah_cas_oop $mem,$newval" %}
317
318 ins_encode %{
319 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
320 noreg /* addr */,
321 $oldval$$Register /* pre_val */,
322 $tmp2$$Register /* tmp */);
323 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
324 noreg, $mem$$Address, $oldval$$Register, $newval$$Register, $tmp1$$Register, $tmp2$$Register,
325 /*exchange*/ true);
326 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
327 $oldval$$Register /* obj */,
328 $mem$$Register /* addr */,
329 noreg /* tmp1 */,
330 $tmp1$$Register /* tmp2 */,
331 $tmp2$$Register /* tmp3 */,
332 false /* narrow */);
333 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
334 $mem$$Register /* addr */,
335 $tmp1$$Register /* addr_tmp */,
336 $tmp2$$Register /* tmp */);
337 %}
338 ins_pipe( pipe_cmpxchg );
339 %}
340
341 instruct getAndSetP_shenandoah(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rFlagsReg cr)
342 %{
343 match(Set newval (GetAndSetP mem newval));
344 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
345 effect(TEMP tmp1, TEMP tmp2, KILL cr);
346 format %{ "xchgq $newval, $mem" %}
347 ins_encode %{
348 assert_different_registers($mem$$Register, $newval$$Register);
349 __ xchgq($newval$$Register, Address($mem$$Register, 0));
350 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
351 noreg /* addr */,
352 $newval$$Register /* pre_val */,
353 $tmp1$$Register /* tmp */);
354 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
355 $newval$$Register /* obj */,
356 $mem$$Register /* addr */,
357 noreg /* tmp1 */,
358 $tmp1$$Register /* tmp2 */,
359 $tmp2$$Register /* tmp3 */,
360 false /* narrow */);
361 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
362 $mem$$Register /* addr */,
363 $tmp1$$Register /* addr_tmp */,
364 $tmp2$$Register /* tmp */);
365 %}
366 ins_pipe(pipe_cmpxchg);
367 %}
368
369 instruct getAndSetN_shenandoah(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
370 %{
371 match(Set newval (GetAndSetN mem newval));
372 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
373 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
374 format %{ "xchgq $newval, $mem" %}
375 ins_encode %{
376 assert_different_registers($mem$$Register, $newval$$Register);
377 __ xchgl($newval$$Register, Address($mem$$Register, 0));
378 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
379 __ movl($tmp1$$Register, $newval$$Register);
380 __ decode_heap_oop($tmp1$$Register);
381 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
382 noreg /* addr */,
383 $tmp1$$Register /* pre_val */,
384 $tmp2$$Register /* tmp */);
385 }
386 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
387 $newval$$Register /* obj */,
388 $mem$$Register /* addr */,
389 $tmp1$$Register /* tmp1 */,
390 $tmp2$$Register /* tmp2 */,
391 $tmp3$$Register /* tmp3 */,
392 true /* narrow */);
393 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
394 $mem$$Register /* addr */,
395 $tmp1$$Register /* addr_tmp */,
396 $tmp2$$Register /* tmp */);
397 %}
398 ins_pipe(pipe_cmpxchg);
399 %}
400