1 //
2 // Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23
24 source_hpp %{
25
26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
27 #include "gc/shared/gc_globals.hpp"
28
29 %}
30
31 source %{
32
33 #include "gc/g1/g1BarrierSetAssembler_x86.hpp"
34 #include "gc/g1/g1BarrierSetRuntime.hpp"
35
36 static void write_barrier_pre(MacroAssembler* masm,
37 const MachNode* node,
38 Register obj,
39 Register pre_val,
40 Register tmp,
41 RegSet preserve = RegSet(),
42 RegSet no_preserve = RegSet()) {
43 if (!G1PreBarrierStubC2::needs_barrier(node)) {
44 return;
45 }
46 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
47 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
48 G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
49 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
50 stub->preserve(*reg);
51 }
52 for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
53 stub->dont_preserve(*reg);
54 }
55 g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, tmp, stub);
56 }
57
58 static void write_barrier_post(MacroAssembler* masm,
59 const MachNode* node,
60 Register store_addr,
61 Register new_val,
62 Register tmp1) {
63 if (!G1BarrierStubC2::needs_post_barrier(node)) {
64 return;
65 }
66 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
67 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
68 bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
69 g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, new_val_may_be_null);
70 }
71
72 %}
73
74 instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
75 %{
76 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
77 match(Set mem (StoreP mem src));
78 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
79 ins_cost(125); // XXX
80 format %{ "movq $mem, $src\t# ptr" %}
81 ins_encode %{
82 // Materialize the store address internally (as opposed to defining 'mem' as
83 // an indirect memory operand) to reduce the overhead of LCM when processing
84 // large basic blocks with many stores. Such basic blocks arise, for
85 // instance, from static initializations of large String arrays.
86 // The same holds for g1StoreN and g1EncodePAndStoreN.
87 __ lea($tmp1$$Register, $mem$$Address);
88 write_barrier_pre(masm, this,
89 $tmp1$$Register /* obj */,
90 $tmp2$$Register /* pre_val */,
91 $tmp3$$Register /* tmp */,
92 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
93 __ movq(Address($tmp1$$Register, 0), $src$$Register);
94 write_barrier_post(masm, this,
95 $tmp1$$Register /* store_addr */,
96 $src$$Register /* new_val */,
97 $tmp3$$Register /* tmp1 */);
98 %}
99 ins_pipe(ialu_mem_reg);
100 %}
101
102 instruct g1StoreN(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
103 %{
104 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
105 match(Set mem (StoreN mem src));
106 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
107 ins_cost(125); // XXX
108 format %{ "movl $mem, $src\t# ptr" %}
109 ins_encode %{
110 __ lea($tmp1$$Register, $mem$$Address);
111 write_barrier_pre(masm, this,
112 $tmp1$$Register /* obj */,
113 $tmp2$$Register /* pre_val */,
114 $tmp3$$Register /* tmp */,
115 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
116 __ movl(Address($tmp1$$Register, 0), $src$$Register);
117 if ((barrier_data() & G1C2BarrierPost) != 0) {
118 __ movl($tmp2$$Register, $src$$Register);
119 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
120 __ decode_heap_oop($tmp2$$Register);
121 } else {
122 __ decode_heap_oop_not_null($tmp2$$Register);
123 }
124 }
125 write_barrier_post(masm, this,
126 $tmp1$$Register /* store_addr */,
127 $tmp2$$Register /* new_val */,
128 $tmp3$$Register /* tmp1 */);
129 %}
130 ins_pipe(ialu_mem_reg);
131 %}
132
133 instruct g1EncodePAndStoreN(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
134 %{
135 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
136 match(Set mem (StoreN mem (EncodeP src)));
137 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
138 ins_cost(125); // XXX
139 format %{ "encode_heap_oop $src\n\t"
140 "movl $mem, $src\t# ptr" %}
141 ins_encode %{
142 __ lea($tmp1$$Register, $mem$$Address);
143 write_barrier_pre(masm, this,
144 $tmp1$$Register /* obj */,
145 $tmp2$$Register /* pre_val */,
146 $tmp3$$Register /* tmp */,
147 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
148 __ movq($tmp2$$Register, $src$$Register);
149 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
150 __ encode_heap_oop($tmp2$$Register);
151 } else {
152 __ encode_heap_oop_not_null($tmp2$$Register);
153 }
154 __ movl(Address($tmp1$$Register, 0), $tmp2$$Register);
155 write_barrier_post(masm, this,
156 $tmp1$$Register /* store_addr */,
157 $src$$Register /* new_val */,
158 $tmp3$$Register /* tmp1 */);
159 %}
160 ins_pipe(ialu_mem_reg);
161 %}
162
163 instruct g1CompareAndExchangeP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
164 %{
165 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
166 match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
167 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
168 format %{ "lock\n\t"
169 "cmpxchgq $newval, $mem" %}
170 ins_encode %{
171 assert_different_registers($oldval$$Register, $mem$$Register);
172 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
173 // $oldval is the only value that can be overwritten.
174 // The same holds for g1CompareAndSwapP.
175 write_barrier_pre(masm, this,
176 noreg /* obj */,
177 $oldval$$Register /* pre_val */,
178 $tmp3$$Register /* tmp */,
179 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
180 __ movq($tmp1$$Register, $newval$$Register);
181 __ lock();
182 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
183 write_barrier_post(masm, this,
184 $mem$$Register /* store_addr */,
185 $tmp1$$Register /* new_val */,
186 $tmp2$$Register /* tmp1 */);
187 %}
188 ins_pipe(pipe_cmpxchg);
189 %}
190
191 instruct g1CompareAndExchangeN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
192 %{
193 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
194 match(Set oldval (CompareAndExchangeN mem (Binary oldval newval)));
195 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
196 format %{ "lock\n\t"
197 "cmpxchgq $newval, $mem" %}
198 ins_encode %{
199 assert_different_registers($oldval$$Register, $mem$$Register);
200 write_barrier_pre(masm, this,
201 $mem$$Register /* obj */,
202 $tmp2$$Register /* pre_val */,
203 $tmp3$$Register /* tmp */,
204 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
205 __ movl($tmp1$$Register, $newval$$Register);
206 __ lock();
207 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
208 __ decode_heap_oop($tmp1$$Register);
209 write_barrier_post(masm, this,
210 $mem$$Register /* store_addr */,
211 $tmp1$$Register /* new_val */,
212 $tmp2$$Register /* tmp1 */);
213 %}
214 ins_pipe(pipe_cmpxchg);
215 %}
216
217 instruct g1CompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
218 %{
219 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
220 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
221 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
222 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
223 format %{ "lock\n\t"
224 "cmpxchgq $newval, $mem\n\t"
225 "sete $res\n\t"
226 "movzbl $res, $res" %}
227 ins_encode %{
228 assert_different_registers($oldval$$Register, $mem$$Register);
229 write_barrier_pre(masm, this,
230 noreg /* obj */,
231 $oldval$$Register /* pre_val */,
232 $tmp3$$Register /* tmp */,
233 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
234 RegSet::of($res$$Register) /* no_preserve */);
235 __ movq($tmp1$$Register, $newval$$Register);
236 __ lock();
237 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
238 __ setb(Assembler::equal, $res$$Register);
239 __ movzbl($res$$Register, $res$$Register);
240 write_barrier_post(masm, this,
241 $mem$$Register /* store_addr */,
242 $tmp1$$Register /* new_val */,
243 $tmp2$$Register /* tmp1 */);
244 %}
245 ins_pipe(pipe_cmpxchg);
246 %}
247
248 instruct g1CompareAndSwapN(rRegI res, indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
249 %{
250 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
251 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
252 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
253 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
254 format %{ "lock\n\t"
255 "cmpxchgq $newval, $mem\n\t"
256 "sete $res\n\t"
257 "movzbl $res, $res" %}
258 ins_encode %{
259 assert_different_registers($oldval$$Register, $mem$$Register);
260 write_barrier_pre(masm, this,
261 $mem$$Register /* obj */,
262 $tmp2$$Register /* pre_val */,
263 $tmp3$$Register /* tmp */,
264 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
265 RegSet::of($res$$Register) /* no_preserve */);
266 __ movl($tmp1$$Register, $newval$$Register);
267 __ lock();
268 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
269 __ setb(Assembler::equal, $res$$Register);
270 __ movzbl($res$$Register, $res$$Register);
271 __ decode_heap_oop($tmp1$$Register);
272 write_barrier_post(masm, this,
273 $mem$$Register /* store_addr */,
274 $tmp1$$Register /* new_val */,
275 $tmp2$$Register /* tmp1 */);
276 %}
277 ins_pipe(pipe_cmpxchg);
278 %}
279
280 instruct g1GetAndSetP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
281 %{
282 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
283 match(Set newval (GetAndSetP mem newval));
284 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
285 format %{ "xchgq $newval, $mem" %}
286 ins_encode %{
287 assert_different_registers($mem$$Register, $newval$$Register);
288 write_barrier_pre(masm, this,
289 $mem$$Register /* obj */,
290 $tmp2$$Register /* pre_val */,
291 $tmp3$$Register /* tmp */,
292 RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
293 __ movq($tmp1$$Register, $newval$$Register);
294 __ xchgq($newval$$Register, Address($mem$$Register, 0));
295 write_barrier_post(masm, this,
296 $mem$$Register /* store_addr */,
297 $tmp1$$Register /* new_val */,
298 $tmp2$$Register /* tmp1 */);
299 %}
300 ins_pipe(pipe_cmpxchg);
301 %}
302
303 instruct g1GetAndSetN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
304 %{
305 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
306 match(Set newval (GetAndSetN mem newval));
307 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
308 format %{ "xchgq $newval, $mem" %}
309 ins_encode %{
310 assert_different_registers($mem$$Register, $newval$$Register);
311 write_barrier_pre(masm, this,
312 $mem$$Register /* obj */,
313 $tmp2$$Register /* pre_val */,
314 $tmp3$$Register /* tmp */,
315 RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
316 __ movl($tmp1$$Register, $newval$$Register);
317 __ decode_heap_oop($tmp1$$Register);
318 __ xchgl($newval$$Register, Address($mem$$Register, 0));
319 write_barrier_post(masm, this,
320 $mem$$Register /* store_addr */,
321 $tmp1$$Register /* new_val */,
322 $tmp2$$Register /* tmp1 */);
323 %}
324 ins_pipe(pipe_cmpxchg);
325 %}
326
327 instruct g1LoadP(rRegP dst, memory mem, rRegP tmp, rFlagsReg cr)
328 %{
329 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
330 match(Set dst (LoadP mem));
331 effect(TEMP dst, TEMP tmp, KILL cr);
332 ins_cost(125); // XXX
333 format %{ "movq $dst, $mem\t# ptr" %}
334 ins_encode %{
335 __ movq($dst$$Register, $mem$$Address);
336 write_barrier_pre(masm, this,
337 noreg /* obj */,
338 $dst$$Register /* pre_val */,
339 $tmp$$Register /* tmp */);
340 %}
341 ins_pipe(ialu_reg_mem); // XXX
342 %}
343
344 instruct g1LoadN(rRegN dst, memory mem, rRegP tmp1, rRegP tmp2, rFlagsReg cr)
345 %{
346 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
347 match(Set dst (LoadN mem));
348 effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
349 ins_cost(125); // XXX
350 format %{ "movl $dst, $mem\t# compressed ptr" %}
351 ins_encode %{
352 __ movl($dst$$Register, $mem$$Address);
353 __ movl($tmp1$$Register, $dst$$Register);
354 __ decode_heap_oop($tmp1$$Register);
355 write_barrier_pre(masm, this,
356 noreg /* obj */,
357 $tmp1$$Register /* pre_val */,
358 $tmp2$$Register /* tmp */);
359 %}
360 ins_pipe(ialu_reg_mem); // XXX
361 %}