1 //
2 // Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23
24 source_hpp %{
25
26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
27 #include "gc/shared/gc_globals.hpp"
28
29 %}
30
31 source %{
32
33 #include "gc/g1/g1BarrierSetAssembler_x86.hpp"
34 #include "gc/g1/g1BarrierSetRuntime.hpp"
35
36 static void write_barrier_pre(MacroAssembler* masm,
37 const MachNode* node,
38 Register obj,
39 Register pre_val,
40 Register tmp,
41 RegSet preserve = RegSet(),
42 RegSet no_preserve = RegSet()) {
43 if (!G1PreBarrierStubC2::needs_barrier(node)) {
44 return;
45 }
46 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
47 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
48 G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
49 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
50 stub->preserve(*reg);
51 }
52 for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
53 stub->dont_preserve(*reg);
54 }
55 g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, tmp, stub);
56 }
57
58 static void write_barrier_post(MacroAssembler* masm,
59 const MachNode* node,
60 Register store_addr,
61 Register new_val,
62 Register tmp1) {
63 if (!G1BarrierStubC2::needs_post_barrier(node)) {
64 return;
65 }
66 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
67 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
68 bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
69 g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, new_val_may_be_null);
70 }
71
72 %}
73
74 instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
75 %{
76 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
77 match(Set mem (StoreP mem src));
78 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
79 ins_cost(125); // XXX
80 format %{ "movq $mem, $src\t# ptr" %}
81 ins_encode %{
82 // Materialize the store address internally (as opposed to defining 'mem' as
83 // an indirect memory operand) to reduce the overhead of LCM when processing
84 // large basic blocks with many stores. Such basic blocks arise, for
85 // instance, from static initializations of large String arrays.
86 // The same holds for g1StoreN and g1EncodePAndStoreN.
87 __ lea($tmp1$$Register, $mem$$Address);
88 write_barrier_pre(masm, this,
89 $tmp1$$Register /* obj */,
90 $tmp2$$Register /* pre_val */,
91 $tmp3$$Register /* tmp */,
92 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
93 __ movq(Address($tmp1$$Register, 0), $src$$Register);
94 write_barrier_post(masm, this,
95 $tmp1$$Register /* store_addr */,
96 $src$$Register /* new_val */,
97 $tmp3$$Register /* tmp1 */);
98 %}
99 ins_pipe(ialu_mem_reg);
100 %}
101
102 // TODO 8350865 (same applies to g1StoreLSpecialTwoOops)
103 // - Do no set/overwrite barrier data here, also handle G1C2BarrierPostNotNull
104 instruct g1StoreLSpecialOneOopOff0(memory mem, rRegL src, immI_0 off, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
105 %{
106 predicate(UseG1GC);
107 match(Set mem (StoreLSpecial mem (Binary src off)));
108 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
109 format %{ "movq $mem, $src\t# g1StoreLSpecialOneOopOff0" %}
110 ins_encode %{
111 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
112
113 __ lea($tmp1$$Register, $mem$$Address);
114 write_barrier_pre(masm, this,
115 $tmp1$$Register /* obj */,
116 $tmp2$$Register /* pre_val */,
117 $tmp3$$Register /* tmp */,
118 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
119
120 __ movq(Address($tmp1$$Register, 0), $src$$Register);
121
122 // Extract the narrow oop field value
123 __ movl($tmp2$$Register, $src$$Register);
124 __ decode_heap_oop($tmp2$$Register);
125 write_barrier_post(masm, this,
126 $tmp1$$Register /* store_addr */,
127 $tmp2$$Register /* new_val */,
128 $tmp3$$Register /* tmp1 */);
129 %}
130 ins_pipe(ialu_mem_reg);
131 %}
132
133 instruct g1StoreLSpecialOneOopOff4(memory mem, rRegL src, immI_4 off, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
134 %{
135 predicate(UseG1GC);
136 match(Set mem (StoreLSpecial mem (Binary src off)));
137 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
138 format %{ "movq $mem, $src\t# g1StoreLSpecialOneOopOff4" %}
139 ins_encode %{
140 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
141
142 // Adjust address to point to narrow oop
143 Address dst = $mem$$Address;
144 __ lea($tmp1$$Register, dst.plus_disp(4));
145 write_barrier_pre(masm, this,
146 $tmp1$$Register /* obj */,
147 $tmp2$$Register /* pre_val */,
148 $tmp3$$Register /* tmp */,
149 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
150
151 // The address of the oop is the address of the store plus the offset of the oop
152 __ movq(Address($tmp1$$Register, -4), $src$$Register);
153
154 // Shift long value to extract the narrow oop field value
155 __ movq($tmp2$$Register, $src$$Register);
156 __ shrq($tmp2$$Register, 32);
157 __ decode_heap_oop($tmp2$$Register);
158 write_barrier_post(masm, this,
159 $tmp1$$Register /* store_addr */,
160 $src$$Register /* new_val */,
161 $tmp3$$Register /* tmp1 */);
162 %}
163 ins_pipe(ialu_mem_reg);
164 %}
165
166 instruct g1StoreLSpecialTwoOops(memory mem, rRegL src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
167 %{
168 predicate(UseG1GC);
169 match(Set mem (StoreLSpecial mem src));
170 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
171 format %{ "movq $mem, $src\t# g1StoreLSpecialTwoOops" %}
172 ins_encode %{
173 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
174
175 __ lea($tmp1$$Register, $mem$$Address);
176 write_barrier_pre(masm, this,
177 $tmp1$$Register /* obj */,
178 $tmp2$$Register /* pre_val */,
179 $tmp3$$Register /* tmp */,
180 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
181 // Adjust address to point to the second narrow oop in the long value
182 __ addq($tmp1$$Register, 4);
183 write_barrier_pre(masm, this,
184 $tmp1$$Register /* obj */,
185 $tmp2$$Register /* pre_val */,
186 $tmp3$$Register /* tmp */,
187 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
188
189 // The address of the second oop is the address of the store plus the offset of the second oop
190 __ movq(Address($tmp1$$Register, -4), $src$$Register);
191
192 // Do the post-barrier of the second oop first since we have its address in tmp1
193 __ movq($tmp2$$Register, $src$$Register);
194 __ shrq($tmp2$$Register, 32);
195 __ decode_heap_oop($tmp2$$Register);
196 write_barrier_post(masm, this,
197 $tmp1$$Register /* store_addr */,
198 $tmp2$$Register /* new_val */,
199 $tmp3$$Register /* tmp1 */);
200
201 // Retrieve the address of the first narrow oop
202 __ addq($tmp1$$Register, -4);
203 // Extract the first narrow oop
204 __ movl($tmp2$$Register, $src$$Register);
205 __ decode_heap_oop($tmp2$$Register);
206 write_barrier_post(masm, this,
207 $tmp1$$Register /* store_addr */,
208 $tmp2$$Register /* new_val */,
209 $tmp3$$Register /* tmp1 */);
210 %}
211 ins_pipe(ialu_mem_reg);
212 %}
213
214 instruct g1StoreN(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
215 %{
216 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
217 match(Set mem (StoreN mem src));
218 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
219 ins_cost(125); // XXX
220 format %{ "movl $mem, $src\t# ptr" %}
221 ins_encode %{
222 __ lea($tmp1$$Register, $mem$$Address);
223 write_barrier_pre(masm, this,
224 $tmp1$$Register /* obj */,
225 $tmp2$$Register /* pre_val */,
226 $tmp3$$Register /* tmp */,
227 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
228 __ movl(Address($tmp1$$Register, 0), $src$$Register);
229 if ((barrier_data() & G1C2BarrierPost) != 0) {
230 __ movl($tmp2$$Register, $src$$Register);
231 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
232 __ decode_heap_oop($tmp2$$Register);
233 } else {
234 __ decode_heap_oop_not_null($tmp2$$Register);
235 }
236 }
237 write_barrier_post(masm, this,
238 $tmp1$$Register /* store_addr */,
239 $tmp2$$Register /* new_val */,
240 $tmp3$$Register /* tmp1 */);
241 %}
242 ins_pipe(ialu_mem_reg);
243 %}
244
245 instruct g1EncodePAndStoreN(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
246 %{
247 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
248 match(Set mem (StoreN mem (EncodeP src)));
249 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
250 ins_cost(125); // XXX
251 format %{ "encode_heap_oop $src\n\t"
252 "movl $mem, $src\t# ptr" %}
253 ins_encode %{
254 __ lea($tmp1$$Register, $mem$$Address);
255 write_barrier_pre(masm, this,
256 $tmp1$$Register /* obj */,
257 $tmp2$$Register /* pre_val */,
258 $tmp3$$Register /* tmp */,
259 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
260 __ movq($tmp2$$Register, $src$$Register);
261 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
262 __ encode_heap_oop($tmp2$$Register);
263 } else {
264 __ encode_heap_oop_not_null($tmp2$$Register);
265 }
266 __ movl(Address($tmp1$$Register, 0), $tmp2$$Register);
267 write_barrier_post(masm, this,
268 $tmp1$$Register /* store_addr */,
269 $src$$Register /* new_val */,
270 $tmp3$$Register /* tmp1 */);
271 %}
272 ins_pipe(ialu_mem_reg);
273 %}
274
275 instruct g1CompareAndExchangeP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
276 %{
277 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
278 match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
279 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
280 format %{ "lock\n\t"
281 "cmpxchgq $newval, $mem" %}
282 ins_encode %{
283 assert_different_registers($oldval$$Register, $mem$$Register);
284 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
285 // $oldval is the only value that can be overwritten.
286 // The same holds for g1CompareAndSwapP.
287 write_barrier_pre(masm, this,
288 noreg /* obj */,
289 $oldval$$Register /* pre_val */,
290 $tmp3$$Register /* tmp */,
291 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
292 __ movq($tmp1$$Register, $newval$$Register);
293 __ lock();
294 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
295 write_barrier_post(masm, this,
296 $mem$$Register /* store_addr */,
297 $tmp1$$Register /* new_val */,
298 $tmp2$$Register /* tmp1 */);
299 %}
300 ins_pipe(pipe_cmpxchg);
301 %}
302
303 instruct g1CompareAndExchangeN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
304 %{
305 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
306 match(Set oldval (CompareAndExchangeN mem (Binary oldval newval)));
307 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
308 format %{ "lock\n\t"
309 "cmpxchgq $newval, $mem" %}
310 ins_encode %{
311 assert_different_registers($oldval$$Register, $mem$$Register);
312 write_barrier_pre(masm, this,
313 $mem$$Register /* obj */,
314 $tmp2$$Register /* pre_val */,
315 $tmp3$$Register /* tmp */,
316 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
317 __ movl($tmp1$$Register, $newval$$Register);
318 __ lock();
319 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
320 __ decode_heap_oop($tmp1$$Register);
321 write_barrier_post(masm, this,
322 $mem$$Register /* store_addr */,
323 $tmp1$$Register /* new_val */,
324 $tmp2$$Register /* tmp1 */);
325 %}
326 ins_pipe(pipe_cmpxchg);
327 %}
328
329 instruct g1CompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
330 %{
331 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
332 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
333 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
334 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
335 format %{ "lock\n\t"
336 "cmpxchgq $newval, $mem\n\t"
337 "sete $res\n\t"
338 "movzbl $res, $res" %}
339 ins_encode %{
340 assert_different_registers($oldval$$Register, $mem$$Register);
341 write_barrier_pre(masm, this,
342 noreg /* obj */,
343 $oldval$$Register /* pre_val */,
344 $tmp3$$Register /* tmp */,
345 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
346 RegSet::of($res$$Register) /* no_preserve */);
347 __ movq($tmp1$$Register, $newval$$Register);
348 __ lock();
349 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
350 __ setb(Assembler::equal, $res$$Register);
351 __ movzbl($res$$Register, $res$$Register);
352 write_barrier_post(masm, this,
353 $mem$$Register /* store_addr */,
354 $tmp1$$Register /* new_val */,
355 $tmp2$$Register /* tmp1 */);
356 %}
357 ins_pipe(pipe_cmpxchg);
358 %}
359
360 instruct g1CompareAndSwapN(rRegI res, indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
361 %{
362 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
363 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
364 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
365 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
366 format %{ "lock\n\t"
367 "cmpxchgq $newval, $mem\n\t"
368 "sete $res\n\t"
369 "movzbl $res, $res" %}
370 ins_encode %{
371 assert_different_registers($oldval$$Register, $mem$$Register);
372 write_barrier_pre(masm, this,
373 $mem$$Register /* obj */,
374 $tmp2$$Register /* pre_val */,
375 $tmp3$$Register /* tmp */,
376 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
377 RegSet::of($res$$Register) /* no_preserve */);
378 __ movl($tmp1$$Register, $newval$$Register);
379 __ lock();
380 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
381 __ setb(Assembler::equal, $res$$Register);
382 __ movzbl($res$$Register, $res$$Register);
383 __ decode_heap_oop($tmp1$$Register);
384 write_barrier_post(masm, this,
385 $mem$$Register /* store_addr */,
386 $tmp1$$Register /* new_val */,
387 $tmp2$$Register /* tmp1 */);
388 %}
389 ins_pipe(pipe_cmpxchg);
390 %}
391
392 instruct g1GetAndSetP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
393 %{
394 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
395 match(Set newval (GetAndSetP mem newval));
396 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
397 format %{ "xchgq $newval, $mem" %}
398 ins_encode %{
399 assert_different_registers($mem$$Register, $newval$$Register);
400 write_barrier_pre(masm, this,
401 $mem$$Register /* obj */,
402 $tmp2$$Register /* pre_val */,
403 $tmp3$$Register /* tmp */,
404 RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
405 __ movq($tmp1$$Register, $newval$$Register);
406 __ xchgq($newval$$Register, Address($mem$$Register, 0));
407 write_barrier_post(masm, this,
408 $mem$$Register /* store_addr */,
409 $tmp1$$Register /* new_val */,
410 $tmp2$$Register /* tmp1 */);
411 %}
412 ins_pipe(pipe_cmpxchg);
413 %}
414
415 instruct g1GetAndSetN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
416 %{
417 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
418 match(Set newval (GetAndSetN mem newval));
419 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
420 format %{ "xchgq $newval, $mem" %}
421 ins_encode %{
422 assert_different_registers($mem$$Register, $newval$$Register);
423 write_barrier_pre(masm, this,
424 $mem$$Register /* obj */,
425 $tmp2$$Register /* pre_val */,
426 $tmp3$$Register /* tmp */,
427 RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
428 __ movl($tmp1$$Register, $newval$$Register);
429 __ decode_heap_oop($tmp1$$Register);
430 __ xchgl($newval$$Register, Address($mem$$Register, 0));
431 write_barrier_post(masm, this,
432 $mem$$Register /* store_addr */,
433 $tmp1$$Register /* new_val */,
434 $tmp2$$Register /* tmp1 */);
435 %}
436 ins_pipe(pipe_cmpxchg);
437 %}
438
439 instruct g1LoadP(rRegP dst, memory mem, rRegP tmp, rFlagsReg cr)
440 %{
441 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
442 match(Set dst (LoadP mem));
443 effect(TEMP dst, TEMP tmp, KILL cr);
444 ins_cost(125); // XXX
445 format %{ "movq $dst, $mem\t# ptr" %}
446 ins_encode %{
447 __ movq($dst$$Register, $mem$$Address);
448 write_barrier_pre(masm, this,
449 noreg /* obj */,
450 $dst$$Register /* pre_val */,
451 $tmp$$Register /* tmp */);
452 %}
453 ins_pipe(ialu_reg_mem); // XXX
454 %}
455
456 instruct g1LoadN(rRegN dst, memory mem, rRegP tmp1, rRegP tmp2, rFlagsReg cr)
457 %{
458 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
459 match(Set dst (LoadN mem));
460 effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
461 ins_cost(125); // XXX
462 format %{ "movl $dst, $mem\t# compressed ptr" %}
463 ins_encode %{
464 __ movl($dst$$Register, $mem$$Address);
465 __ movl($tmp1$$Register, $dst$$Register);
466 __ decode_heap_oop($tmp1$$Register);
467 write_barrier_pre(masm, this,
468 noreg /* obj */,
469 $tmp1$$Register /* pre_val */,
470 $tmp2$$Register /* tmp */);
471 %}
472 ins_pipe(ialu_reg_mem); // XXX
473 %}