1 //
2 // Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23
24 source_hpp %{
25
26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
27 #include "gc/shared/gc_globals.hpp"
28
29 %}
30
31 source %{
32
33 #include "gc/g1/g1BarrierSetAssembler_x86.hpp"
34 #include "gc/g1/g1BarrierSetRuntime.hpp"
35
36 static void write_barrier_pre(MacroAssembler* masm,
37 const MachNode* node,
38 Register obj,
39 Register pre_val,
40 Register tmp,
41 RegSet preserve = RegSet(),
42 RegSet no_preserve = RegSet()) {
43 if (!G1PreBarrierStubC2::needs_barrier(node)) {
44 return;
45 }
46 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
47 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
48 G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
49 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
50 stub->preserve(*reg);
51 }
52 for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
53 stub->dont_preserve(*reg);
54 }
55 g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, r15_thread, tmp, stub);
56 }
57
58 static void write_barrier_post(MacroAssembler* masm,
59 const MachNode* node,
60 Register store_addr,
61 Register new_val,
62 Register tmp1,
63 Register tmp2) {
64 if (!G1PostBarrierStubC2::needs_barrier(node)) {
65 return;
66 }
67 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
68 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
69 G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
70 g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, r15_thread, tmp1, tmp2, stub);
71 }
72
73 %}
74
75 instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
76 %{
77 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
78 match(Set mem (StoreP mem src));
79 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
80 ins_cost(125); // XXX
81 format %{ "movq $mem, $src\t# ptr" %}
82 ins_encode %{
83 // Materialize the store address internally (as opposed to defining 'mem' as
84 // an indirect memory operand) to reduce the overhead of LCM when processing
85 // large basic blocks with many stores. Such basic blocks arise, for
86 // instance, from static initializations of large String arrays.
87 // The same holds for g1StoreN and g1EncodePAndStoreN.
88 __ lea($tmp1$$Register, $mem$$Address);
89 write_barrier_pre(masm, this,
90 $tmp1$$Register /* obj */,
91 $tmp2$$Register /* pre_val */,
92 $tmp3$$Register /* tmp */,
93 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
94 __ movq(Address($tmp1$$Register, 0), $src$$Register);
95 write_barrier_post(masm, this,
96 $tmp1$$Register /* store_addr */,
97 $src$$Register /* new_val */,
98 $tmp3$$Register /* tmp1 */,
99 $tmp2$$Register /* tmp2 */);
100 %}
101 ins_pipe(ialu_mem_reg);
102 %}
103
104 instruct g1StoreN(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
105 %{
106 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
107 match(Set mem (StoreN mem src));
108 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
109 ins_cost(125); // XXX
110 format %{ "movl $mem, $src\t# ptr" %}
111 ins_encode %{
112 __ lea($tmp1$$Register, $mem$$Address);
113 write_barrier_pre(masm, this,
114 $tmp1$$Register /* obj */,
115 $tmp2$$Register /* pre_val */,
116 $tmp3$$Register /* tmp */,
117 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
118 __ movl(Address($tmp1$$Register, 0), $src$$Register);
119 if ((barrier_data() & G1C2BarrierPost) != 0) {
120 __ movl($tmp2$$Register, $src$$Register);
121 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
122 __ decode_heap_oop($tmp2$$Register);
123 } else {
124 __ decode_heap_oop_not_null($tmp2$$Register);
125 }
126 }
127 write_barrier_post(masm, this,
128 $tmp1$$Register /* store_addr */,
129 $tmp2$$Register /* new_val */,
130 $tmp3$$Register /* tmp1 */,
131 $tmp2$$Register /* tmp2 */);
132 %}
133 ins_pipe(ialu_mem_reg);
134 %}
135
136 instruct g1EncodePAndStoreN(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
137 %{
138 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
139 match(Set mem (StoreN mem (EncodeP src)));
140 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
141 ins_cost(125); // XXX
142 format %{ "encode_heap_oop $src\n\t"
143 "movl $mem, $src\t# ptr" %}
144 ins_encode %{
145 __ lea($tmp1$$Register, $mem$$Address);
146 write_barrier_pre(masm, this,
147 $tmp1$$Register /* obj */,
148 $tmp2$$Register /* pre_val */,
149 $tmp3$$Register /* tmp */,
150 RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
151 __ movq($tmp2$$Register, $src$$Register);
152 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
153 __ encode_heap_oop($tmp2$$Register);
154 } else {
155 __ encode_heap_oop_not_null($tmp2$$Register);
156 }
157 __ movl(Address($tmp1$$Register, 0), $tmp2$$Register);
158 write_barrier_post(masm, this,
159 $tmp1$$Register /* store_addr */,
160 $src$$Register /* new_val */,
161 $tmp3$$Register /* tmp1 */,
162 $tmp2$$Register /* tmp2 */);
163 %}
164 ins_pipe(ialu_mem_reg);
165 %}
166
167 instruct g1CompareAndExchangeP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
168 %{
169 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
170 match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
171 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
172 format %{ "lock\n\t"
173 "cmpxchgq $newval, $mem" %}
174 ins_encode %{
175 assert_different_registers($oldval$$Register, $mem$$Register);
176 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
177 // $oldval is the only value that can be overwritten.
178 // The same holds for g1CompareAndSwapP.
179 write_barrier_pre(masm, this,
180 noreg /* obj */,
181 $oldval$$Register /* pre_val */,
182 $tmp3$$Register /* tmp */,
183 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
184 __ movq($tmp1$$Register, $newval$$Register);
185 __ lock();
186 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
187 write_barrier_post(masm, this,
188 $mem$$Register /* store_addr */,
189 $tmp1$$Register /* new_val */,
190 $tmp2$$Register /* tmp1 */,
191 $tmp3$$Register /* tmp2 */);
192 %}
193 ins_pipe(pipe_cmpxchg);
194 %}
195
196 instruct g1CompareAndExchangeN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
197 %{
198 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
199 match(Set oldval (CompareAndExchangeN mem (Binary oldval newval)));
200 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
201 format %{ "lock\n\t"
202 "cmpxchgq $newval, $mem" %}
203 ins_encode %{
204 assert_different_registers($oldval$$Register, $mem$$Register);
205 write_barrier_pre(masm, this,
206 $mem$$Register /* obj */,
207 $tmp2$$Register /* pre_val */,
208 $tmp3$$Register /* tmp */,
209 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
210 __ movl($tmp1$$Register, $newval$$Register);
211 __ lock();
212 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
213 __ decode_heap_oop($tmp1$$Register);
214 write_barrier_post(masm, this,
215 $mem$$Register /* store_addr */,
216 $tmp1$$Register /* new_val */,
217 $tmp2$$Register /* tmp1 */,
218 $tmp3$$Register /* tmp2 */);
219 %}
220 ins_pipe(pipe_cmpxchg);
221 %}
222
223 instruct g1CompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
224 %{
225 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
226 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
227 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
228 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
229 format %{ "lock\n\t"
230 "cmpxchgq $newval, $mem\n\t"
231 "sete $res\n\t"
232 "movzbl $res, $res" %}
233 ins_encode %{
234 assert_different_registers($oldval$$Register, $mem$$Register);
235 write_barrier_pre(masm, this,
236 noreg /* obj */,
237 $oldval$$Register /* pre_val */,
238 $tmp3$$Register /* tmp */,
239 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
240 RegSet::of($res$$Register) /* no_preserve */);
241 __ movq($tmp1$$Register, $newval$$Register);
242 __ lock();
243 __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
244 __ setb(Assembler::equal, $res$$Register);
245 __ movzbl($res$$Register, $res$$Register);
246 write_barrier_post(masm, this,
247 $mem$$Register /* store_addr */,
248 $tmp1$$Register /* new_val */,
249 $tmp2$$Register /* tmp1 */,
250 $tmp3$$Register /* tmp2 */);
251 %}
252 ins_pipe(pipe_cmpxchg);
253 %}
254
255 instruct g1CompareAndSwapN(rRegI res, indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
256 %{
257 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
258 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
259 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
260 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
261 format %{ "lock\n\t"
262 "cmpxchgq $newval, $mem\n\t"
263 "sete $res\n\t"
264 "movzbl $res, $res" %}
265 ins_encode %{
266 assert_different_registers($oldval$$Register, $mem$$Register);
267 write_barrier_pre(masm, this,
268 $mem$$Register /* obj */,
269 $tmp2$$Register /* pre_val */,
270 $tmp3$$Register /* tmp */,
271 RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
272 RegSet::of($res$$Register) /* no_preserve */);
273 __ movl($tmp1$$Register, $newval$$Register);
274 __ lock();
275 __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
276 __ setb(Assembler::equal, $res$$Register);
277 __ movzbl($res$$Register, $res$$Register);
278 __ decode_heap_oop($tmp1$$Register);
279 write_barrier_post(masm, this,
280 $mem$$Register /* store_addr */,
281 $tmp1$$Register /* new_val */,
282 $tmp2$$Register /* tmp1 */,
283 $tmp3$$Register /* tmp2 */);
284 %}
285 ins_pipe(pipe_cmpxchg);
286 %}
287
288 instruct g1GetAndSetP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
289 %{
290 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
291 match(Set newval (GetAndSetP mem newval));
292 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
293 format %{ "xchgq $newval, $mem" %}
294 ins_encode %{
295 assert_different_registers($mem$$Register, $newval$$Register);
296 write_barrier_pre(masm, this,
297 $mem$$Register /* obj */,
298 $tmp2$$Register /* pre_val */,
299 $tmp3$$Register /* tmp */,
300 RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
301 __ movq($tmp1$$Register, $newval$$Register);
302 __ xchgq($newval$$Register, Address($mem$$Register, 0));
303 write_barrier_post(masm, this,
304 $mem$$Register /* store_addr */,
305 $tmp1$$Register /* new_val */,
306 $tmp2$$Register /* tmp1 */,
307 $tmp3$$Register /* tmp2 */);
308 %}
309 ins_pipe(pipe_cmpxchg);
310 %}
311
312 instruct g1GetAndSetN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
313 %{
314 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
315 match(Set newval (GetAndSetN mem newval));
316 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
317 format %{ "xchgq $newval, $mem" %}
318 ins_encode %{
319 assert_different_registers($mem$$Register, $newval$$Register);
320 write_barrier_pre(masm, this,
321 $mem$$Register /* obj */,
322 $tmp2$$Register /* pre_val */,
323 $tmp3$$Register /* tmp */,
324 RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
325 __ movl($tmp1$$Register, $newval$$Register);
326 __ decode_heap_oop($tmp1$$Register);
327 __ xchgl($newval$$Register, Address($mem$$Register, 0));
328 write_barrier_post(masm, this,
329 $mem$$Register /* store_addr */,
330 $tmp1$$Register /* new_val */,
331 $tmp2$$Register /* tmp1 */,
332 $tmp3$$Register /* tmp2 */);
333 %}
334 ins_pipe(pipe_cmpxchg);
335 %}
336
337 instruct g1LoadP(rRegP dst, memory mem, rRegP tmp, rFlagsReg cr)
338 %{
339 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
340 match(Set dst (LoadP mem));
341 effect(TEMP dst, TEMP tmp, KILL cr);
342 ins_cost(125); // XXX
343 format %{ "movq $dst, $mem\t# ptr" %}
344 ins_encode %{
345 __ movq($dst$$Register, $mem$$Address);
346 write_barrier_pre(masm, this,
347 noreg /* obj */,
348 $dst$$Register /* pre_val */,
349 $tmp$$Register /* tmp */);
350 %}
351 ins_pipe(ialu_reg_mem); // XXX
352 %}
353
354 instruct g1LoadN(rRegN dst, memory mem, rRegP tmp1, rRegP tmp2, rFlagsReg cr)
355 %{
356 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
357 match(Set dst (LoadN mem));
358 effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
359 ins_cost(125); // XXX
360 format %{ "movl $dst, $mem\t# compressed ptr" %}
361 ins_encode %{
362 __ movl($dst$$Register, $mem$$Address);
363 __ movl($tmp1$$Register, $dst$$Register);
364 __ decode_heap_oop($tmp1$$Register);
365 write_barrier_pre(masm, this,
366 noreg /* obj */,
367 $tmp1$$Register /* pre_val */,
368 $tmp2$$Register /* tmp */);
369 %}
370 ins_pipe(ialu_reg_mem); // XXX
371 %}