1 //
2 // Copyright (c) 2018, Red Hat, Inc. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 source %{
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 %}
30
31 instruct storeP_shenandoah(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
32 %{
33 match(Set mem (StoreP mem src));
34 predicate(UseShenandoahGC && n->as_Store()->barrier_data() != 0);
35 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
36 ins_cost(125); // XXX
37 format %{ "movq $mem, $src\t# ptr" %}
38 ins_encode %{
39 __ lea($tmp1$$Register, $mem$$Address);
40 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
41 $tmp1$$Register /* addr */,
42 $tmp2$$Register /* pre_val */,
43 $tmp3$$Register /* tmp */);
44 __ movq(Address($tmp1$$Register, 0), $src$$Register);
45 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
46 noreg /* addr */,
47 $tmp1$$Register /* addr_tmp */,
48 $tmp2$$Register /* tmp */);
49 %}
50 ins_pipe(ialu_mem_reg);
51 %}
52
53 instruct storeN_shenandoah(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
54 %{
55 match(Set mem (StoreN mem src));
56 predicate(UseShenandoahGC && n->as_Store()->barrier_data() != 0);
57 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
58 ins_cost(125); // XXX
59 format %{ "movl $mem, $src\t# ptr" %}
60 ins_encode %{
61 __ lea($tmp1$$Register, $mem$$Address);
62 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
63 $tmp1$$Register /* addr */,
64 $tmp2$$Register /* pre_val */,
65 $tmp3$$Register /* tmp */);
66 __ movl(Address($tmp1$$Register, 0), $src$$Register);
67 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
68 noreg /* addr */,
69 $tmp1$$Register /* addr_tmp */,
70 $tmp2$$Register /* tmp */);
71 %}
72 ins_pipe(ialu_mem_reg);
73 %}
74
75 instruct encodePAndStoreN_shenandoah(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
76 %{
77 match(Set mem (StoreN mem (EncodeP src)));
78 predicate(UseShenandoahGC && n->as_Store()->barrier_data() != 0);
79 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
80 ins_cost(125); // XXX
81 format %{ "encode_heap_oop $src\n\t"
82 "movl $mem, $src\t# ptr" %}
83 ins_encode %{
84 __ lea($tmp1$$Register, $mem$$Address);
85 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
86 $tmp1$$Register /* addr */,
87 $tmp2$$Register /* pre_val */,
88 $tmp3$$Register /* tmp */);
89 __ movq($tmp3$$Register, $src$$Register);
90 if ((barrier_data() & ShenandoahBarrierCardMarkNotNull) == 0) {
91 __ encode_heap_oop($tmp3$$Register);
92 } else {
93 __ encode_heap_oop_not_null($tmp3$$Register);
94 }
95 __ movl(Address($tmp1$$Register, 0), $tmp3$$Register);
96 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
97 noreg /* addr */,
98 $tmp1$$Register /* addr_tmp */,
99 $tmp2$$Register /* tmp */);
100 %}
101 ins_pipe(ialu_mem_reg);
102 %}
103
104 instruct compareAndSwapP_shenandoah(rRegI res,
105 indirect mem,
106 rRegP tmp1, rRegP tmp2,
107 rax_RegP oldval, rRegP newval,
108 rFlagsReg cr)
109 %{
110 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
111 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
112 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
113 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
114
115 format %{ "shenandoah_cas_oop $mem,$newval" %}
116
117 ins_encode %{
118 guarantee(!UseCompressedOops, "must not be compressed oops");
119 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
120 __ movptr($tmp1$$Register, $oldval$$Register);
121 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
122 noreg /* addr */,
123 $tmp1$$Register /* pre_val */,
124 $tmp2$$Register /* tmp */);
125 }
126 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
127 $res$$Register, $mem$$Address, $oldval$$Register, $newval$$Register, $tmp1$$Register, $tmp2$$Register,
128 /*exchange*/ false);
129 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
130 $mem$$Register /* addr */,
131 $tmp1$$Register /* addr_tmp */,
132 $tmp2$$Register /* tmp */);
133 %}
134 ins_pipe( pipe_cmpxchg );
135 %}
136
137 instruct compareAndSwapN_shenandoah(rRegI res,
138 indirect mem,
139 rRegP tmp1, rRegP tmp2,
140 rax_RegN oldval, rRegN newval,
141 rFlagsReg cr) %{
142 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
143 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
144 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
145 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
146
147 format %{ "shenandoah_cas_oop $mem,$newval" %}
148
149 ins_encode %{
150 guarantee(UseCompressedOops, "must be compressed oops");
151 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
152 __ movl($tmp1$$Register, $oldval$$Register);
153 __ decode_heap_oop($tmp1$$Register);
154 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
155 noreg /* addr */,
156 $tmp1$$Register /* pre_val */,
157 $tmp2$$Register /* tmp */);
158 }
159 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
160 $res$$Register, $mem$$Address, $oldval$$Register, $newval$$Register, $tmp1$$Register, $tmp2$$Register,
161 /*exchange*/ false);
162 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
163 $mem$$Register /* addr */,
164 $tmp1$$Register /* addr_tmp */,
165 $tmp2$$Register /* tmp */);
166 %}
167 ins_pipe( pipe_cmpxchg );
168 %}
169
170 instruct compareAndExchangeN_shenandoah(indirect mem,
171 rax_RegN oldval, rRegN newval,
172 rRegP tmp1, rRegP tmp2, rRegP tmp3,
173 rFlagsReg cr) %{
174 match(Set oldval (CompareAndExchangeN mem (Binary oldval newval)));
175 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
176 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
177
178 format %{ "shenandoah_cas_oop $mem,$newval" %}
179
180 ins_encode %{
181 guarantee(UseCompressedOops, "must be compressed oops");
182 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
183 __ movl($tmp1$$Register, $oldval$$Register);
184 __ decode_heap_oop($tmp1$$Register);
185 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
186 noreg /* addr */,
187 $tmp1$$Register /* pre_val */,
188 $tmp2$$Register /* tmp */);
189 }
190 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
191 noreg, $mem$$Address, $oldval$$Register, $newval$$Register, $tmp1$$Register, $tmp2$$Register,
192 /*exchange*/ true);
193 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
194 $oldval$$Register /* obj */,
195 $mem$$Register /* addr */,
196 $tmp1$$Register /* tmp1 */,
197 $tmp2$$Register /* tmp2 */,
198 $tmp3$$Register /* tmp3 */,
199 true /* narrow */);
200 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
201 $mem$$Register /* addr */,
202 $tmp1$$Register /* addr_tmp */,
203 $tmp2$$Register /* tmp */);
204 %}
205 ins_pipe( pipe_cmpxchg );
206 %}
207
208 instruct compareAndExchangeP_shenandoah(indirect mem,
209 rax_RegP oldval, rRegP newval,
210 rRegP tmp1, rRegP tmp2,
211 rFlagsReg cr)
212 %{
213 match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
214 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
215 effect(KILL cr, TEMP tmp1, TEMP tmp2);
216 ins_cost(1000);
217
218 format %{ "shenandoah_cas_oop $mem,$newval" %}
219
220 ins_encode %{
221 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
222 noreg /* addr */,
223 $oldval$$Register /* pre_val */,
224 $tmp2$$Register /* tmp */);
225 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
226 noreg, $mem$$Address, $oldval$$Register, $newval$$Register, $tmp1$$Register, $tmp2$$Register,
227 /*exchange*/ true);
228 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
229 $oldval$$Register /* obj */,
230 $mem$$Register /* addr */,
231 noreg /* tmp1 */,
232 $tmp1$$Register /* tmp2 */,
233 $tmp2$$Register /* tmp3 */,
234 false /* narrow */);
235 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
236 $mem$$Register /* addr */,
237 $tmp1$$Register /* addr_tmp */,
238 $tmp2$$Register /* tmp */);
239 %}
240 ins_pipe( pipe_cmpxchg );
241 %}
242
243 instruct getAndSetP_shenandoah(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rFlagsReg cr)
244 %{
245 match(Set newval (GetAndSetP mem newval));
246 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
247 effect(TEMP tmp1, TEMP tmp2, KILL cr);
248 format %{ "xchgq $newval, $mem" %}
249 ins_encode %{
250 assert_different_registers($mem$$Register, $newval$$Register);
251 __ xchgq($newval$$Register, Address($mem$$Register, 0));
252 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
253 noreg /* addr */,
254 $newval$$Register /* pre_val */,
255 $tmp1$$Register /* tmp */);
256 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
257 $newval$$Register /* obj */,
258 $mem$$Register /* addr */,
259 noreg /* tmp1 */,
260 $tmp1$$Register /* tmp2 */,
261 $tmp2$$Register /* tmp3 */,
262 false /* narrow */);
263 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
264 $mem$$Register /* addr */,
265 $tmp1$$Register /* addr_tmp */,
266 $tmp2$$Register /* tmp */);
267 %}
268 ins_pipe(pipe_cmpxchg);
269 %}
270
271 instruct getAndSetN_shenandoah(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
272 %{
273 match(Set newval (GetAndSetN mem newval));
274 predicate(UseShenandoahGC && n->as_LoadStore()->barrier_data() != 0);
275 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
276 format %{ "xchgq $newval, $mem" %}
277 ins_encode %{
278 assert_different_registers($mem$$Register, $newval$$Register);
279 __ xchgl($newval$$Register, Address($mem$$Register, 0));
280 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
281 __ movl($tmp1$$Register, $newval$$Register);
282 __ decode_heap_oop($tmp1$$Register);
283 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
284 noreg /* addr */,
285 $tmp1$$Register /* pre_val */,
286 $tmp2$$Register /* tmp */);
287 }
288 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
289 $newval$$Register /* obj */,
290 $mem$$Register /* addr */,
291 $tmp1$$Register /* tmp1 */,
292 $tmp2$$Register /* tmp2 */,
293 $tmp3$$Register /* tmp3 */,
294 true /* narrow */);
295 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
296 $mem$$Register /* addr */,
297 $tmp1$$Register /* addr_tmp */,
298 $tmp2$$Register /* tmp */);
299 %}
300 ins_pipe(pipe_cmpxchg);
301 %}
302
303 instruct loadP_shenandaoh(rRegP dst, memory mem, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
304 %{
305 match(Set dst (LoadP mem));
306 predicate(UseShenandoahGC && n->as_Load()->barrier_data() != 0);
307 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
308 ins_cost(125); // XXX
309 format %{ "movq $dst, $mem\t# ptr" %}
310 ins_encode %{
311 __ movq($dst$$Register, $mem$$Address);
312 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
313 noreg /* addr */,
314 $dst$$Register /* pre_val */,
315 $tmp1$$Register /* tmp */);
316 __ lea($tmp3$$Register, $mem$$Address);
317 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
318 $dst$$Register /* obj */,
319 $tmp3$$Register /* addr */,
320 noreg /* tmp1 */,
321 $tmp1$$Register /* tmp2 */,
322 $tmp2$$Register /* tmp3 */,
323 false /* narrow */);
324 %}
325 ins_pipe(ialu_reg_mem); // XXX
326 %}
327
328 instruct loadN_shenandoah(rRegN dst, indirect mem, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
329 %{
330 match(Set dst (LoadN mem));
331 predicate(UseShenandoahGC && n->as_Load()->barrier_data() != 0);
332 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
333 ins_cost(125); // XXX
334 format %{ "movl $dst, $mem\t# compressed ptr" %}
335 ins_encode %{
336 __ movl($dst$$Register, $mem$$Address);
337 if (ShenandoahSATBBarrierStubC2::needs_barrier(this)) {
338 __ movl($tmp1$$Register, $dst$$Register);
339 __ decode_heap_oop($tmp1$$Register);
340 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
341 noreg /* addr */,
342 $tmp1$$Register /* pre_val */,
343 $tmp2$$Register /* tmp */);
344 }
345 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
346 $dst$$Register /* obj */,
347 $mem$$Register /* addr */,
348 $tmp1$$Register /* tmp1 */,
349 $tmp2$$Register /* tmp2 */,
350 $tmp3$$Register /* tmp3 */,
351 true /* narrow */);
352 %}
353 ins_pipe(ialu_reg_mem); // XXX
354 %}