1 //
2 // Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23
24 source_hpp %{
25
26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
27 #include "gc/shared/gc_globals.hpp"
28
29 %}
30
31 source %{
32
33 #include "gc/g1/g1BarrierSetAssembler_x86.hpp"
34 #include "gc/g1/g1BarrierSetRuntime.hpp"
35
36 static void write_barrier_pre(MacroAssembler* masm,
37 const MachNode* node,
38 Register obj,
39 Register pre_val,
40 Register tmp,
41 RegSet preserve = RegSet(),
42 RegSet no_preserve = RegSet()) {
43 if (!G1PreBarrierStubC2::needs_barrier(node)) {
44 return;
45 }
46 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
47 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
48 G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
49 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
50 stub->preserve(*reg);
51 }
52 for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
53 stub->dont_preserve(*reg);
54 }
55 g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, r15_thread, tmp, stub);
56 }
57
58 static void write_barrier_post(MacroAssembler* masm,
59 const MachNode* node,
60 Register store_addr,
61 Register new_val,
62 Register tmp1,
63 Register tmp2,
64 RegSet preserve = RegSet()) {
65 if (!G1PostBarrierStubC2::needs_barrier(node)) {
66 return;
67 }
68 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
69 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
70 G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
71 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
72 stub->preserve(*reg);
73 }
74 g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, r15_thread, tmp1, tmp2, stub);
75 }
76
77 %}
78
79 instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
80 %{
81 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
82 match(Set mem (StoreP mem src));
83 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
84 ins_cost(125); // XXX
85 format %{ "movq $mem, $src\t# ptr" %}
86 ins_encode %{
87 // Materialize the store address internally (as opposed to defining 'mem' as
88 // an indirect memory operand) to reduce the overhead of LCM when processing
89 // large basic blocks with many stores. Such basic blocks arise, for
90 // instance, from static initializations of large String arrays.
91 // The same holds for g1StoreN and g1EncodePAndStoreN.
92 __ lea($tmp1$$Register, $mem$$Address);
93 write_barrier_pre(masm, this,
94 $tmp1$$Register /* obj */,
95 $tmp2$$Register /* pre_val */,
96 $tmp3$$Register /* tmp */,
97 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
98 __ movq(Address($tmp1$$Register, 0), $src$$Register);
99 write_barrier_post(masm, this,
100 $tmp1$$Register /* store_addr */,
101 $src$$Register /* new_val */,
102 $tmp3$$Register /* tmp1 */,
103 $tmp2$$Register /* tmp2 */);
104 %}
105 ins_pipe(ialu_mem_reg);
106 %}
107
108 // TODO 8350865 (same applies to g1StoreLSpecialTwoOops)
109 // - Can we use an unbound register for src?
110 // - Do no set/overwrite barrier data here, also handle G1C2BarrierPostNotNull
111 // - Is the zero-extend really required in all the places?
112 instruct g1StoreLSpecialOneOop(memory mem, rdx_RegL src, immI off, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
113 %{
114 predicate(UseG1GC);
115 match(Set mem (StoreLSpecial mem (Binary src off)));
116 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL src, KILL cr);
117 format %{ "movq $mem, $src\t# g1StoreLSpecialOneOop" %}
118 ins_encode %{
119 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
120
121 __ lea($tmp1$$Register, $mem$$Address);
122 // Adjust address to point to narrow oop
123 __ addq($tmp1$$Register, $off$$constant);
124 write_barrier_pre(masm, this,
125 $tmp1$$Register /* obj */,
126 $tmp2$$Register /* pre_val */,
127 $tmp3$$Register /* tmp */,
128 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
129
130 __ movq($mem$$Address, $src$$Register);
131
132 // Shift long value to extract the narrow oop field value and zero-extend it
133 __ shrq($src$$Register, $off$$constant << LogBitsPerByte);
134 __ movl($src$$Register, $src$$Register);
135
136 write_barrier_post(masm, this,
137 $tmp1$$Register /* store_addr */,
138 $src$$Register /* new_val */,
139 $tmp3$$Register /* tmp1 */,
140 $tmp2$$Register /* tmp2 */);
141 %}
142 ins_pipe(ialu_mem_reg);
143 %}
144
145 instruct g1StoreLSpecialTwoOops(memory mem, rdx_RegL src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rRegP tmp4, rFlagsReg cr)
146 %{
147 predicate(UseG1GC);
148 match(Set mem (StoreLSpecial mem src));
149 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, KILL cr);
150 format %{ "movq $mem, $src\t# g1StoreLSpecialTwoOops" %}
151 ins_encode %{
152 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
153
154 __ lea($tmp1$$Register, $mem$$Address);
155 write_barrier_pre(masm, this,
156 $tmp1$$Register /* obj */,
157 $tmp2$$Register /* pre_val */,
158 $tmp3$$Register /* tmp */,
159 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
160 // Adjust address to point to the second narrow oop in the long value
161 __ addq($tmp1$$Register, 4);
162 write_barrier_pre(masm, this,
163 $tmp1$$Register /* obj */,
164 $tmp2$$Register /* pre_val */,
165 $tmp3$$Register /* tmp */,
166 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
167
168 __ movq($mem$$Address, $src$$Register);
169
170 // Zero-extend first narrow oop to long
171 __ movl($tmp4$$Register, $src$$Register);
172
173 // Shift long value to extract the second narrow oop field value
174 __ shrq($src$$Register, 32);
175
176 write_barrier_post(masm, this,
177 $tmp1$$Register /* store_addr */,
178 $src$$Register /* new_val */,
179 $tmp3$$Register /* tmp1 */,
180 $tmp2$$Register /* tmp2 */,
181 RegSet::of($tmp1$$Register, $tmp4$$Register) /* preserve */);
182 // Adjust address again to point to the first narrow oop in the long value
183 __ subq($tmp1$$Register, 4);
184 write_barrier_post(masm, this,
185 $tmp1$$Register /* store_addr */,
186 $tmp4$$Register /* new_val */,
187 $tmp3$$Register /* tmp1 */,
188 $tmp2$$Register /* tmp2 */);
189 %}
190 ins_pipe(ialu_mem_reg);
191 %}
192
193 instruct g1StoreN(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
194 %{
195 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
196 match(Set mem (StoreN mem src));
197 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
198 ins_cost(125); // XXX
199 format %{ "movl $mem, $src\t# ptr" %}
200 ins_encode %{
201 __ lea($tmp1$$Register, $mem$$Address);
202 write_barrier_pre(masm, this,
203 $tmp1$$Register /* obj */,
204 $tmp2$$Register /* pre_val */,
205 $tmp3$$Register /* tmp */,
206 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
207 __ movl(Address($tmp1$$Register, 0), $src$$Register);
208 if ((barrier_data() & G1C2BarrierPost) != 0) {
209 __ movl($tmp2$$Register, $src$$Register);
210 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
211 __ decode_heap_oop($tmp2$$Register);
212 } else {
213 __ decode_heap_oop_not_null($tmp2$$Register);
214 }
215 }
216 write_barrier_post(masm, this,
217 $tmp1$$Register /* store_addr */,
218 $tmp2$$Register /* new_val */,
219 $tmp3$$Register /* tmp1 */,
220 $tmp2$$Register /* tmp2 */);
221 %}
222 ins_pipe(ialu_mem_reg);
223 %}
224
225 instruct g1EncodePAndStoreN(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
226 %{
227 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
228 match(Set mem (StoreN mem (EncodeP src)));
229 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
230 ins_cost(125); // XXX
231 format %{ "encode_heap_oop $src\n\t"
232 "movl $mem, $src\t# ptr" %}
233 ins_encode %{
234 __ lea($tmp1$$Register, $mem$$Address);
235 write_barrier_pre(masm, this,
236 $tmp1$$Register /* obj */,
237 $tmp2$$Register /* pre_val */,
238 $tmp3$$Register /* tmp */,
239 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
240 __ movq($tmp2$$Register, $src$$Register);
241 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
242 __ encode_heap_oop($tmp2$$Register);
243 } else {
244 __ encode_heap_oop_not_null($tmp2$$Register);
245 }
246 __ movl(Address($tmp1$$Register, 0), $tmp2$$Register);
247 write_barrier_post(masm, this,
248 $tmp1$$Register /* store_addr */,
249 $src$$Register /* new_val */,
250 $tmp3$$Register /* tmp1 */,
251 $tmp2$$Register /* tmp2 */);
252 %}
253 ins_pipe(ialu_mem_reg);
254 %}
255
256 instruct g1CompareAndExchangeP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
257 %{
258 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
259 match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
260 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
261 format %{ "lock\n\t"
262 "cmpxchgq $newval, $mem" %}
263 ins_encode %{
264 assert_different_registers($oldval$$Register, $mem$$Register);
265 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
266 // $oldval is the only value that can be overwritten.
267 // The same holds for g1CompareAndSwapP.
268 write_barrier_pre(masm, this,
269 noreg /* obj */,
270 $oldval$$Register /* pre_val */,
271 $tmp3$$Register /* tmp */,
272 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
273 __ movq($tmp1$$Register, $newval$$Register);
274 __ lock();
275 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
276 write_barrier_post(masm, this,
277 $mem$$Register /* store_addr */,
278 $tmp1$$Register /* new_val */,
279 $tmp2$$Register /* tmp1 */,
280 $tmp3$$Register /* tmp2 */);
281 %}
282 ins_pipe(pipe_cmpxchg);
283 %}
284
285 instruct g1CompareAndExchangeN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
286 %{
287 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
288 match(Set oldval (CompareAndExchangeN mem (Binary oldval newval)));
289 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
290 format %{ "lock\n\t"
291 "cmpxchgq $newval, $mem" %}
292 ins_encode %{
293 assert_different_registers($oldval$$Register, $mem$$Register);
294 write_barrier_pre(masm, this,
295 $mem$$Register /* obj */,
296 $tmp2$$Register /* pre_val */,
297 $tmp3$$Register /* tmp */,
298 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
299 __ movl($tmp1$$Register, $newval$$Register);
300 __ lock();
301 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
302 __ decode_heap_oop($tmp1$$Register);
303 write_barrier_post(masm, this,
304 $mem$$Register /* store_addr */,
305 $tmp1$$Register /* new_val */,
306 $tmp2$$Register /* tmp1 */,
307 $tmp3$$Register /* tmp2 */);
308 %}
309 ins_pipe(pipe_cmpxchg);
310 %}
311
312 instruct g1CompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
313 %{
314 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
315 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
316 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
317 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
318 format %{ "lock\n\t"
319 "cmpxchgq $newval, $mem\n\t"
320 "sete $res\n\t"
321 "movzbl $res, $res" %}
322 ins_encode %{
323 assert_different_registers($oldval$$Register, $mem$$Register);
324 write_barrier_pre(masm, this,
325 noreg /* obj */,
326 $oldval$$Register /* pre_val */,
327 $tmp3$$Register /* tmp */,
328 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
329 RegSet::of($res$$Register) /* no_preserve */);
330 __ movq($tmp1$$Register, $newval$$Register);
331 __ lock();
332 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
333 __ setb(Assembler::equal, $res$$Register);
334 __ movzbl($res$$Register, $res$$Register);
335 write_barrier_post(masm, this,
336 $mem$$Register /* store_addr */,
337 $tmp1$$Register /* new_val */,
338 $tmp2$$Register /* tmp1 */,
339 $tmp3$$Register /* tmp2 */);
340 %}
341 ins_pipe(pipe_cmpxchg);
342 %}
343
344 instruct g1CompareAndSwapN(rRegI res, indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
345 %{
346 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
347 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
348 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
349 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
350 format %{ "lock\n\t"
351 "cmpxchgq $newval, $mem\n\t"
352 "sete $res\n\t"
353 "movzbl $res, $res" %}
354 ins_encode %{
355 assert_different_registers($oldval$$Register, $mem$$Register);
356 write_barrier_pre(masm, this,
357 $mem$$Register /* obj */,
358 $tmp2$$Register /* pre_val */,
359 $tmp3$$Register /* tmp */,
360 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
361 RegSet::of($res$$Register) /* no_preserve */);
362 __ movl($tmp1$$Register, $newval$$Register);
363 __ lock();
364 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
365 __ setb(Assembler::equal, $res$$Register);
366 __ movzbl($res$$Register, $res$$Register);
367 __ decode_heap_oop($tmp1$$Register);
368 write_barrier_post(masm, this,
369 $mem$$Register /* store_addr */,
370 $tmp1$$Register /* new_val */,
371 $tmp2$$Register /* tmp1 */,
372 $tmp3$$Register /* tmp2 */);
373 %}
374 ins_pipe(pipe_cmpxchg);
375 %}
376
377 instruct g1GetAndSetP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
378 %{
379 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
380 match(Set newval (GetAndSetP mem newval));
381 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
382 format %{ "xchgq $newval, $mem" %}
383 ins_encode %{
384 assert_different_registers($mem$$Register, $newval$$Register);
385 write_barrier_pre(masm, this,
386 $mem$$Register /* obj */,
387 $tmp2$$Register /* pre_val */,
388 $tmp3$$Register /* tmp */,
389 RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
390 __ movq($tmp1$$Register, $newval$$Register);
391 __ xchgq($newval$$Register, Address($mem$$Register, 0));
392 write_barrier_post(masm, this,
393 $mem$$Register /* store_addr */,
394 $tmp1$$Register /* new_val */,
395 $tmp2$$Register /* tmp1 */,
396 $tmp3$$Register /* tmp2 */);
397 %}
398 ins_pipe(pipe_cmpxchg);
399 %}
400
401 instruct g1GetAndSetN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
402 %{
403 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
404 match(Set newval (GetAndSetN mem newval));
405 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
406 format %{ "xchgq $newval, $mem" %}
407 ins_encode %{
408 assert_different_registers($mem$$Register, $newval$$Register);
409 write_barrier_pre(masm, this,
410 $mem$$Register /* obj */,
411 $tmp2$$Register /* pre_val */,
412 $tmp3$$Register /* tmp */,
413 RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
414 __ movl($tmp1$$Register, $newval$$Register);
415 __ decode_heap_oop($tmp1$$Register);
416 __ xchgl($newval$$Register, Address($mem$$Register, 0));
417 write_barrier_post(masm, this,
418 $mem$$Register /* store_addr */,
419 $tmp1$$Register /* new_val */,
420 $tmp2$$Register /* tmp1 */,
421 $tmp3$$Register /* tmp2 */);
422 %}
423 ins_pipe(pipe_cmpxchg);
424 %}
425
426 instruct g1LoadP(rRegP dst, memory mem, rRegP tmp, rFlagsReg cr)
427 %{
428 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
429 match(Set dst (LoadP mem));
430 effect(TEMP dst, TEMP tmp, KILL cr);
431 ins_cost(125); // XXX
432 format %{ "movq $dst, $mem\t# ptr" %}
433 ins_encode %{
434 __ movq($dst$$Register, $mem$$Address);
435 write_barrier_pre(masm, this,
436 noreg /* obj */,
437 $dst$$Register /* pre_val */,
438 $tmp$$Register /* tmp */);
439 %}
440 ins_pipe(ialu_reg_mem); // XXX
441 %}
442
443 instruct g1LoadN(rRegN dst, memory mem, rRegP tmp1, rRegP tmp2, rFlagsReg cr)
444 %{
445 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
446 match(Set dst (LoadN mem));
447 effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
448 ins_cost(125); // XXX
449 format %{ "movl $dst, $mem\t# compressed ptr" %}
450 ins_encode %{
451 __ movl($dst$$Register, $mem$$Address);
452 __ movl($tmp1$$Register, $dst$$Register);
453 __ decode_heap_oop($tmp1$$Register);
454 write_barrier_pre(masm, this,
455 noreg /* obj */,
456 $tmp1$$Register /* pre_val */,
457 $tmp2$$Register /* tmp */);
458 %}
459 ins_pipe(ialu_reg_mem); // XXX
460 %}