1 //
2 // Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23
24 source_hpp %{
25
26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
27 #include "gc/shared/gc_globals.hpp"
28
29 %}
30
31 source %{
32
33 #include "gc/g1/g1BarrierSetAssembler_x86.hpp"
34 #include "gc/g1/g1BarrierSetRuntime.hpp"
35
36 static void write_barrier_pre(MacroAssembler* masm,
37 const MachNode* node,
38 Register obj,
39 Register pre_val,
40 Register tmp,
41 RegSet preserve = RegSet(),
42 RegSet no_preserve = RegSet()) {
43 if (!G1PreBarrierStubC2::needs_barrier(node)) {
44 return;
45 }
46 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
47 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
48 G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
49 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
50 stub->preserve(*reg);
51 }
52 for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
53 stub->dont_preserve(*reg);
54 }
55 g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, tmp, stub);
56 }
57
58 static void write_barrier_post(MacroAssembler* masm,
59 const MachNode* node,
60 Register store_addr,
61 Register new_val,
62 Register tmp1) {
63 if (!G1BarrierStubC2::needs_post_barrier(node)) {
64 return;
65 }
66 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
67 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
68 bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
69 g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, new_val_may_be_null);
70 }
71
72 %}
73
74 instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
75 %{
76 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
77 match(Set mem (StoreP mem src));
78 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
79 ins_cost(125); // XXX
80 format %{ "movq $mem, $src\t# ptr" %}
81 ins_encode %{
82 // Materialize the store address internally (as opposed to defining 'mem' as
83 // an indirect memory operand) to reduce the overhead of LCM when processing
84 // large basic blocks with many stores. Such basic blocks arise, for
85 // instance, from static initializations of large String arrays.
86 // The same holds for g1StoreN and g1EncodePAndStoreN.
87 __ lea($tmp1$$Register, $mem$$Address);
88 write_barrier_pre(masm, this,
89 $tmp1$$Register /* obj */,
90 $tmp2$$Register /* pre_val */,
91 $tmp3$$Register /* tmp */,
92 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
93 __ movq(Address($tmp1$$Register, 0), $src$$Register);
94 write_barrier_post(masm, this,
95 $tmp1$$Register /* store_addr */,
96 $src$$Register /* new_val */,
97 $tmp3$$Register /* tmp1 */);
98 %}
99 ins_pipe(ialu_mem_reg);
100 %}
101
102 // TODO 8350865 (same applies to g1StoreLSpecialTwoOops)
103 // - Can we use an unbound register for src?
104 // - Do no set/overwrite barrier data here, also handle G1C2BarrierPostNotNull
105 // - Is the zero-extend really required in all the places?
106 instruct g1StoreLSpecialOneOop(memory mem, rdx_RegL src, immI off, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
107 %{
108 predicate(UseG1GC);
109 match(Set mem (StoreLSpecial mem (Binary src off)));
110 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL src, KILL cr);
111 format %{ "movq $mem, $src\t# g1StoreLSpecialOneOop" %}
112 ins_encode %{
113 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
114
115 __ lea($tmp1$$Register, $mem$$Address);
116 // Adjust address to point to narrow oop
117 __ addq($tmp1$$Register, $off$$constant);
118 write_barrier_pre(masm, this,
119 $tmp1$$Register /* obj */,
120 $tmp2$$Register /* pre_val */,
121 $tmp3$$Register /* tmp */,
122 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
123
124 __ movq(Address($tmp1$$Register, 0), $src$$Register);
125
126 // Shift long value to extract the narrow oop field value and zero-extend it
127 __ shrq($src$$Register, $off$$constant << LogBitsPerByte);
128 __ movl($src$$Register, $src$$Register);
129
130 write_barrier_post(masm, this,
131 $tmp1$$Register /* store_addr */,
132 $src$$Register /* new_val */,
133 $tmp3$$Register /* tmp1 */);
134 %}
135 ins_pipe(ialu_mem_reg);
136 %}
137
138 instruct g1StoreLSpecialTwoOops(memory mem, rdx_RegL src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rRegP tmp4, rFlagsReg cr)
139 %{
140 predicate(UseG1GC);
141 match(Set mem (StoreLSpecial mem src));
142 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, KILL cr);
143 format %{ "movq $mem, $src\t# g1StoreLSpecialTwoOops" %}
144 ins_encode %{
145 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
146
147 __ lea($tmp1$$Register, $mem$$Address);
148 write_barrier_pre(masm, this,
149 $tmp1$$Register /* obj */,
150 $tmp2$$Register /* pre_val */,
151 $tmp3$$Register /* tmp */,
152 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
153 // Adjust address to point to the second narrow oop in the long value
154 __ addq($tmp1$$Register, 4);
155 write_barrier_pre(masm, this,
156 $tmp1$$Register /* obj */,
157 $tmp2$$Register /* pre_val */,
158 $tmp3$$Register /* tmp */,
159 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
160 // Adjust address again to point to the first narrow oop in the long value
161 __ subq($tmp1$$Register, 4);
162
163 __ movq(Address($tmp1$$Register, 0), $src$$Register);
164
165 // Zero-extend first narrow oop to long
166 __ movl($tmp4$$Register, $src$$Register);
167
168 // Shift long value to extract the second narrow oop field value
169 __ shrq($src$$Register, 32);
170 write_barrier_post(masm, this,
171 $tmp1$$Register /* store_addr */,
172 $tmp4$$Register /* new_val */,
173 $tmp3$$Register /* tmp1 */);
174 __ addq($tmp1$$Register, 4);
175 write_barrier_post(masm, this,
176 $tmp1$$Register /* store_addr */,
177 $src$$Register /* new_val */,
178 $tmp3$$Register /* tmp1 */);
179 %}
180 ins_pipe(ialu_mem_reg);
181 %}
182
183 instruct g1StoreN(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
184 %{
185 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
186 match(Set mem (StoreN mem src));
187 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
188 ins_cost(125); // XXX
189 format %{ "movl $mem, $src\t# ptr" %}
190 ins_encode %{
191 __ lea($tmp1$$Register, $mem$$Address);
192 write_barrier_pre(masm, this,
193 $tmp1$$Register /* obj */,
194 $tmp2$$Register /* pre_val */,
195 $tmp3$$Register /* tmp */,
196 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
197 __ movl(Address($tmp1$$Register, 0), $src$$Register);
198 if ((barrier_data() & G1C2BarrierPost) != 0) {
199 __ movl($tmp2$$Register, $src$$Register);
200 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
201 __ decode_heap_oop($tmp2$$Register);
202 } else {
203 __ decode_heap_oop_not_null($tmp2$$Register);
204 }
205 }
206 write_barrier_post(masm, this,
207 $tmp1$$Register /* store_addr */,
208 $tmp2$$Register /* new_val */,
209 $tmp3$$Register /* tmp1 */);
210 %}
211 ins_pipe(ialu_mem_reg);
212 %}
213
214 instruct g1EncodePAndStoreN(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
215 %{
216 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
217 match(Set mem (StoreN mem (EncodeP src)));
218 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
219 ins_cost(125); // XXX
220 format %{ "encode_heap_oop $src\n\t"
221 "movl $mem, $src\t# ptr" %}
222 ins_encode %{
223 __ lea($tmp1$$Register, $mem$$Address);
224 write_barrier_pre(masm, this,
225 $tmp1$$Register /* obj */,
226 $tmp2$$Register /* pre_val */,
227 $tmp3$$Register /* tmp */,
228 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
229 __ movq($tmp2$$Register, $src$$Register);
230 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
231 __ encode_heap_oop($tmp2$$Register);
232 } else {
233 __ encode_heap_oop_not_null($tmp2$$Register);
234 }
235 __ movl(Address($tmp1$$Register, 0), $tmp2$$Register);
236 write_barrier_post(masm, this,
237 $tmp1$$Register /* store_addr */,
238 $src$$Register /* new_val */,
239 $tmp3$$Register /* tmp1 */);
240 %}
241 ins_pipe(ialu_mem_reg);
242 %}
243
244 instruct g1CompareAndExchangeP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
245 %{
246 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
247 match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
248 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
249 format %{ "lock\n\t"
250 "cmpxchgq $newval, $mem" %}
251 ins_encode %{
252 assert_different_registers($oldval$$Register, $mem$$Register);
253 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
254 // $oldval is the only value that can be overwritten.
255 // The same holds for g1CompareAndSwapP.
256 write_barrier_pre(masm, this,
257 noreg /* obj */,
258 $oldval$$Register /* pre_val */,
259 $tmp3$$Register /* tmp */,
260 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
261 __ movq($tmp1$$Register, $newval$$Register);
262 __ lock();
263 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
264 write_barrier_post(masm, this,
265 $mem$$Register /* store_addr */,
266 $tmp1$$Register /* new_val */,
267 $tmp2$$Register /* tmp1 */);
268 %}
269 ins_pipe(pipe_cmpxchg);
270 %}
271
272 instruct g1CompareAndExchangeN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
273 %{
274 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
275 match(Set oldval (CompareAndExchangeN mem (Binary oldval newval)));
276 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
277 format %{ "lock\n\t"
278 "cmpxchgq $newval, $mem" %}
279 ins_encode %{
280 assert_different_registers($oldval$$Register, $mem$$Register);
281 write_barrier_pre(masm, this,
282 $mem$$Register /* obj */,
283 $tmp2$$Register /* pre_val */,
284 $tmp3$$Register /* tmp */,
285 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
286 __ movl($tmp1$$Register, $newval$$Register);
287 __ lock();
288 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
289 __ decode_heap_oop($tmp1$$Register);
290 write_barrier_post(masm, this,
291 $mem$$Register /* store_addr */,
292 $tmp1$$Register /* new_val */,
293 $tmp2$$Register /* tmp1 */);
294 %}
295 ins_pipe(pipe_cmpxchg);
296 %}
297
298 instruct g1CompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
299 %{
300 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
301 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
302 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
303 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
304 format %{ "lock\n\t"
305 "cmpxchgq $newval, $mem\n\t"
306 "sete $res\n\t"
307 "movzbl $res, $res" %}
308 ins_encode %{
309 assert_different_registers($oldval$$Register, $mem$$Register);
310 write_barrier_pre(masm, this,
311 noreg /* obj */,
312 $oldval$$Register /* pre_val */,
313 $tmp3$$Register /* tmp */,
314 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
315 RegSet::of($res$$Register) /* no_preserve */);
316 __ movq($tmp1$$Register, $newval$$Register);
317 __ lock();
318 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
319 __ setb(Assembler::equal, $res$$Register);
320 __ movzbl($res$$Register, $res$$Register);
321 write_barrier_post(masm, this,
322 $mem$$Register /* store_addr */,
323 $tmp1$$Register /* new_val */,
324 $tmp2$$Register /* tmp1 */);
325 %}
326 ins_pipe(pipe_cmpxchg);
327 %}
328
329 instruct g1CompareAndSwapN(rRegI res, indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
330 %{
331 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
332 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
333 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
334 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
335 format %{ "lock\n\t"
336 "cmpxchgq $newval, $mem\n\t"
337 "sete $res\n\t"
338 "movzbl $res, $res" %}
339 ins_encode %{
340 assert_different_registers($oldval$$Register, $mem$$Register);
341 write_barrier_pre(masm, this,
342 $mem$$Register /* obj */,
343 $tmp2$$Register /* pre_val */,
344 $tmp3$$Register /* tmp */,
345 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
346 RegSet::of($res$$Register) /* no_preserve */);
347 __ movl($tmp1$$Register, $newval$$Register);
348 __ lock();
349 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
350 __ setb(Assembler::equal, $res$$Register);
351 __ movzbl($res$$Register, $res$$Register);
352 __ decode_heap_oop($tmp1$$Register);
353 write_barrier_post(masm, this,
354 $mem$$Register /* store_addr */,
355 $tmp1$$Register /* new_val */,
356 $tmp2$$Register /* tmp1 */);
357 %}
358 ins_pipe(pipe_cmpxchg);
359 %}
360
361 instruct g1GetAndSetP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
362 %{
363 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
364 match(Set newval (GetAndSetP mem newval));
365 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
366 format %{ "xchgq $newval, $mem" %}
367 ins_encode %{
368 assert_different_registers($mem$$Register, $newval$$Register);
369 write_barrier_pre(masm, this,
370 $mem$$Register /* obj */,
371 $tmp2$$Register /* pre_val */,
372 $tmp3$$Register /* tmp */,
373 RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
374 __ movq($tmp1$$Register, $newval$$Register);
375 __ xchgq($newval$$Register, Address($mem$$Register, 0));
376 write_barrier_post(masm, this,
377 $mem$$Register /* store_addr */,
378 $tmp1$$Register /* new_val */,
379 $tmp2$$Register /* tmp1 */);
380 %}
381 ins_pipe(pipe_cmpxchg);
382 %}
383
384 instruct g1GetAndSetN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
385 %{
386 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
387 match(Set newval (GetAndSetN mem newval));
388 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
389 format %{ "xchgq $newval, $mem" %}
390 ins_encode %{
391 assert_different_registers($mem$$Register, $newval$$Register);
392 write_barrier_pre(masm, this,
393 $mem$$Register /* obj */,
394 $tmp2$$Register /* pre_val */,
395 $tmp3$$Register /* tmp */,
396 RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
397 __ movl($tmp1$$Register, $newval$$Register);
398 __ decode_heap_oop($tmp1$$Register);
399 __ xchgl($newval$$Register, Address($mem$$Register, 0));
400 write_barrier_post(masm, this,
401 $mem$$Register /* store_addr */,
402 $tmp1$$Register /* new_val */,
403 $tmp2$$Register /* tmp1 */);
404 %}
405 ins_pipe(pipe_cmpxchg);
406 %}
407
408 instruct g1LoadP(rRegP dst, memory mem, rRegP tmp, rFlagsReg cr)
409 %{
410 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
411 match(Set dst (LoadP mem));
412 effect(TEMP dst, TEMP tmp, KILL cr);
413 ins_cost(125); // XXX
414 format %{ "movq $dst, $mem\t# ptr" %}
415 ins_encode %{
416 __ movq($dst$$Register, $mem$$Address);
417 write_barrier_pre(masm, this,
418 noreg /* obj */,
419 $dst$$Register /* pre_val */,
420 $tmp$$Register /* tmp */);
421 %}
422 ins_pipe(ialu_reg_mem); // XXX
423 %}
424
425 instruct g1LoadN(rRegN dst, memory mem, rRegP tmp1, rRegP tmp2, rFlagsReg cr)
426 %{
427 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
428 match(Set dst (LoadN mem));
429 effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
430 ins_cost(125); // XXX
431 format %{ "movl $dst, $mem\t# compressed ptr" %}
432 ins_encode %{
433 __ movl($dst$$Register, $mem$$Address);
434 __ movl($tmp1$$Register, $dst$$Register);
435 __ decode_heap_oop($tmp1$$Register);
436 write_barrier_pre(masm, this,
437 noreg /* obj */,
438 $tmp1$$Register /* pre_val */,
439 $tmp2$$Register /* tmp */);
440 %}
441 ins_pipe(ialu_reg_mem); // XXX
442 %}