1 //
  2 // Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
  3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4 //
  5 // This code is free software; you can redistribute it and/or modify it
  6 // under the terms of the GNU General Public License version 2 only, as
  7 // published by the Free Software Foundation.
  8 //
  9 // This code is distributed in the hope that it will be useful, but WITHOUT
 10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12 // version 2 for more details (a copy is included in the LICENSE file that
 13 // accompanied this code).
 14 //
 15 // You should have received a copy of the GNU General Public License version
 16 // 2 along with this work; if not, write to the Free Software Foundation,
 17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18 //
 19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20 // or visit www.oracle.com if you need additional information or have any
 21 // questions.
 22 //
 23 
 24 source_hpp %{
 25 
 26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
 27 #include "gc/shared/gc_globals.hpp"
 28 
 29 %}
 30 
 31 source %{
 32 
 33 #include "gc/g1/g1BarrierSetAssembler_x86.hpp"
 34 #include "gc/g1/g1BarrierSetRuntime.hpp"
 35 
 36 static void write_barrier_pre(MacroAssembler* masm,
 37                               const MachNode* node,
 38                               Register obj,
 39                               Register pre_val,
 40                               Register tmp,
 41                               RegSet preserve = RegSet(),
 42                               RegSet no_preserve = RegSet()) {
 43   if (!G1PreBarrierStubC2::needs_barrier(node)) {
 44     return;
 45   }
 46   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 47   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
 48   G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
 49   for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
 50     stub->preserve(*reg);
 51   }
 52   for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
 53     stub->dont_preserve(*reg);
 54   }
 55   g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, tmp, stub);
 56 }
 57 
 58 static void write_barrier_post(MacroAssembler* masm,
 59                                const MachNode* node,
 60                                Register store_addr,
 61                                Register new_val,
 62                                Register tmp1) {
 63   if (!G1BarrierStubC2::needs_post_barrier(node)) {
 64     return;
 65   }
 66   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 67   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
 68   bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
 69   g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, new_val_may_be_null);
 70 }
 71 
 72 %}
 73 
 74 instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
 75 %{
 76   predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
 77   match(Set mem (StoreP mem src));
 78   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
 79   ins_cost(125); // XXX
 80   format %{ "movq    $mem, $src\t# ptr" %}
 81   ins_encode %{
 82     // Materialize the store address internally (as opposed to defining 'mem' as
 83     // an indirect memory operand) to reduce the overhead of LCM when processing
 84     // large basic blocks with many stores. Such basic blocks arise, for
 85     // instance, from static initializations of large String arrays.
 86     // The same holds for g1StoreN and g1EncodePAndStoreN.
 87     __ lea($tmp1$$Register, $mem$$Address);
 88     write_barrier_pre(masm, this,
 89                       $tmp1$$Register /* obj */,
 90                       $tmp2$$Register /* pre_val */,
 91                       $tmp3$$Register /* tmp */,
 92                       RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
 93     __ movq(Address($tmp1$$Register, 0), $src$$Register);
 94     write_barrier_post(masm, this,
 95                        $tmp1$$Register /* store_addr */,
 96                        $src$$Register /* new_val */,
 97                        $tmp3$$Register /* tmp1 */);
 98   %}
 99   ins_pipe(ialu_mem_reg);
100 %}
101 
102 // TODO 8350865 (same applies to g1StoreLSpecialTwoOops)
103 // - Do no set/overwrite barrier data here, also handle G1C2BarrierPostNotNull
104 instruct g1StoreLSpecialOneOopOff0(memory mem, rRegL src, immI_0 off, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
105 %{
106   predicate(UseG1GC);
107   match(Set mem (StoreLSpecial mem (Binary src off)));
108   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
109   format %{ "movq    $mem, $src\t# g1StoreLSpecialOneOopOff0" %}
110   ins_encode %{
111     ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
112 
113     __ lea($tmp1$$Register, $mem$$Address);
114     write_barrier_pre(masm, this,
115                       $tmp1$$Register /* obj */,
116                       $tmp2$$Register /* pre_val */,
117                       $tmp3$$Register /* tmp */,
118                       RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
119 
120     __ movq(Address($tmp1$$Register, 0), $src$$Register);
121 
122     // Extract the narrow oop field value
123     __ movl($tmp2$$Register, $src$$Register);
124     __ decode_heap_oop($tmp2$$Register);
125     write_barrier_post(masm, this,
126                        $tmp1$$Register /* store_addr */,
127                        $tmp2$$Register /* new_val */,
128                        $tmp3$$Register /* tmp1 */);
129   %}
130   ins_pipe(ialu_mem_reg);
131 %}
132 
133 instruct g1StoreLSpecialOneOopOff4(memory mem, rRegL src, immI_4 off, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
134 %{
135   predicate(UseG1GC);
136   match(Set mem (StoreLSpecial mem (Binary src off)));
137   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
138   format %{ "movq    $mem, $src\t# g1StoreLSpecialOneOopOff4" %}
139   ins_encode %{
140     ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
141 
142     // Adjust address to point to narrow oop
143     Address dst = $mem$$Address;
144     __ lea($tmp1$$Register, dst.plus_disp(4));
145     write_barrier_pre(masm, this,
146                       $tmp1$$Register /* obj */,
147                       $tmp2$$Register /* pre_val */,
148                       $tmp3$$Register /* tmp */,
149                       RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
150 
151     // The address of the oop is the address of the store plus the offset of the oop
152     __ movq(Address($tmp1$$Register, -4), $src$$Register);
153 
154     // Shift long value to extract the narrow oop field value
155     __ movq($tmp2$$Register, $src$$Register);
156     __ shrq($tmp2$$Register, 32);
157     __ decode_heap_oop($tmp2$$Register);
158     write_barrier_post(masm, this,
159                        $tmp1$$Register /* store_addr */,
160                        $src$$Register /* new_val */,
161                        $tmp3$$Register /* tmp1 */);
162   %}
163   ins_pipe(ialu_mem_reg);
164 %}
165 
166 instruct g1StoreLSpecialTwoOops(memory mem, rRegL src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
167 %{
168   predicate(UseG1GC);
169   match(Set mem (StoreLSpecial mem src));
170   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
171   format %{ "movq    $mem, $src\t# g1StoreLSpecialTwoOops" %}
172   ins_encode %{
173     ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
174 
175     __ lea($tmp1$$Register, $mem$$Address);
176     write_barrier_pre(masm, this,
177                       $tmp1$$Register /* obj */,
178                       $tmp2$$Register /* pre_val */,
179                       $tmp3$$Register /* tmp */,
180                       RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
181     // Adjust address to point to the second narrow oop in the long value
182     __ addq($tmp1$$Register, 4);
183     write_barrier_pre(masm, this,
184                       $tmp1$$Register /* obj */,
185                       $tmp2$$Register /* pre_val */,
186                       $tmp3$$Register /* tmp */,
187                       RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
188 
189     // The address of the second oop is the address of the store plus the offset of the second oop
190     __ movq(Address($tmp1$$Register, -4), $src$$Register);
191 
192     // Do the post-barrier of the second oop first since we have its address in tmp1
193     __ movq($tmp2$$Register, $src$$Register);
194     __ shrq($tmp2$$Register, 32);
195     __ decode_heap_oop($tmp2$$Register);
196     write_barrier_post(masm, this,
197                        $tmp1$$Register /* store_addr */,
198                        $tmp2$$Register /* new_val */,
199                        $tmp3$$Register /* tmp1 */);
200 
201     // Retrieve the address of the first narrow oop
202     __ addq($tmp1$$Register, -4);
203     // Extract the first narrow oop
204     __ movl($tmp2$$Register, $src$$Register);
205     __ decode_heap_oop($tmp2$$Register);
206     write_barrier_post(masm, this,
207                        $tmp1$$Register /* store_addr */,
208                        $tmp2$$Register /* new_val */,
209                        $tmp3$$Register /* tmp1 */);
210   %}
211   ins_pipe(ialu_mem_reg);
212 %}
213 
214 instruct g1StoreN(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
215 %{
216   predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
217   match(Set mem (StoreN mem src));
218   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
219   ins_cost(125); // XXX
220   format %{ "movl    $mem, $src\t# ptr" %}
221   ins_encode %{
222     __ lea($tmp1$$Register, $mem$$Address);
223     write_barrier_pre(masm, this,
224                       $tmp1$$Register /* obj */,
225                       $tmp2$$Register /* pre_val */,
226                       $tmp3$$Register /* tmp */,
227                       RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
228     __ movl(Address($tmp1$$Register, 0), $src$$Register);
229     if ((barrier_data() & G1C2BarrierPost) != 0) {
230       __ movl($tmp2$$Register, $src$$Register);
231       if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
232         __ decode_heap_oop($tmp2$$Register);
233       } else {
234         __ decode_heap_oop_not_null($tmp2$$Register);
235       }
236     }
237     write_barrier_post(masm, this,
238                        $tmp1$$Register /* store_addr */,
239                        $tmp2$$Register /* new_val */,
240                        $tmp3$$Register /* tmp1 */);
241   %}
242   ins_pipe(ialu_mem_reg);
243 %}
244 
245 instruct g1EncodePAndStoreN(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
246 %{
247   predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
248   match(Set mem (StoreN mem (EncodeP src)));
249   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
250   ins_cost(125); // XXX
251   format %{ "encode_heap_oop $src\n\t"
252             "movl   $mem, $src\t# ptr" %}
253   ins_encode %{
254     __ lea($tmp1$$Register, $mem$$Address);
255     write_barrier_pre(masm, this,
256                       $tmp1$$Register /* obj */,
257                       $tmp2$$Register /* pre_val */,
258                       $tmp3$$Register /* tmp */,
259                       RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
260     __ movq($tmp2$$Register, $src$$Register);
261     if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
262       __ encode_heap_oop($tmp2$$Register);
263     } else {
264       __ encode_heap_oop_not_null($tmp2$$Register);
265     }
266     __ movl(Address($tmp1$$Register, 0), $tmp2$$Register);
267     write_barrier_post(masm, this,
268                        $tmp1$$Register /* store_addr */,
269                        $src$$Register /* new_val */,
270                        $tmp3$$Register /* tmp1 */);
271   %}
272   ins_pipe(ialu_mem_reg);
273 %}
274 
275 instruct g1CompareAndExchangeP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
276 %{
277   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
278   match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
279   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
280   format %{ "lock\n\t"
281             "cmpxchgq $newval, $mem" %}
282   ins_encode %{
283     assert_different_registers($oldval$$Register, $mem$$Register);
284     // Pass $oldval to the pre-barrier (instead of loading from $mem), because
285     // $oldval is the only value that can be overwritten.
286     // The same holds for g1CompareAndSwapP.
287     write_barrier_pre(masm, this,
288                       noreg /* obj */,
289                       $oldval$$Register /* pre_val */,
290                       $tmp3$$Register /* tmp */,
291                       RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
292     __ movq($tmp1$$Register, $newval$$Register);
293     __ lock();
294     __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
295     write_barrier_post(masm, this,
296                        $mem$$Register /* store_addr */,
297                        $tmp1$$Register /* new_val */,
298                        $tmp2$$Register /* tmp1 */);
299   %}
300   ins_pipe(pipe_cmpxchg);
301 %}
302 
303 instruct g1CompareAndExchangeN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
304 %{
305   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
306   match(Set oldval (CompareAndExchangeN mem (Binary oldval newval)));
307   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
308   format %{ "lock\n\t"
309             "cmpxchgq $newval, $mem" %}
310   ins_encode %{
311     assert_different_registers($oldval$$Register, $mem$$Register);
312     write_barrier_pre(masm, this,
313                       $mem$$Register /* obj */,
314                       $tmp2$$Register /* pre_val */,
315                       $tmp3$$Register /* tmp */,
316                       RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
317     __ movl($tmp1$$Register, $newval$$Register);
318     __ lock();
319     __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
320     __ decode_heap_oop($tmp1$$Register);
321     write_barrier_post(masm, this,
322                        $mem$$Register /* store_addr */,
323                        $tmp1$$Register /* new_val */,
324                        $tmp2$$Register /* tmp1 */);
325   %}
326   ins_pipe(pipe_cmpxchg);
327 %}
328 
329 instruct g1CompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
330 %{
331   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
332   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
333   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
334   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
335   format %{ "lock\n\t"
336             "cmpxchgq $newval, $mem\n\t"
337             "sete     $res\n\t"
338             "movzbl   $res, $res" %}
339   ins_encode %{
340     assert_different_registers($oldval$$Register, $mem$$Register);
341     write_barrier_pre(masm, this,
342                       noreg /* obj */,
343                       $oldval$$Register /* pre_val */,
344                       $tmp3$$Register /* tmp */,
345                       RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
346                       RegSet::of($res$$Register) /* no_preserve */);
347     __ movq($tmp1$$Register, $newval$$Register);
348     __ lock();
349     __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
350     __ setb(Assembler::equal, $res$$Register);
351     __ movzbl($res$$Register, $res$$Register);
352     write_barrier_post(masm, this,
353                        $mem$$Register /* store_addr */,
354                        $tmp1$$Register /* new_val */,
355                        $tmp2$$Register /* tmp1 */);
356   %}
357   ins_pipe(pipe_cmpxchg);
358 %}
359 
360 instruct g1CompareAndSwapN(rRegI res, indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
361 %{
362   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
363   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
364   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
365   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
366   format %{ "lock\n\t"
367             "cmpxchgq $newval, $mem\n\t"
368             "sete     $res\n\t"
369             "movzbl   $res, $res" %}
370   ins_encode %{
371     assert_different_registers($oldval$$Register, $mem$$Register);
372     write_barrier_pre(masm, this,
373                       $mem$$Register /* obj */,
374                       $tmp2$$Register /* pre_val */,
375                       $tmp3$$Register /* tmp */,
376                       RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
377                       RegSet::of($res$$Register) /* no_preserve */);
378     __ movl($tmp1$$Register, $newval$$Register);
379     __ lock();
380     __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
381     __ setb(Assembler::equal, $res$$Register);
382     __ movzbl($res$$Register, $res$$Register);
383     __ decode_heap_oop($tmp1$$Register);
384     write_barrier_post(masm, this,
385                        $mem$$Register /* store_addr */,
386                        $tmp1$$Register /* new_val */,
387                        $tmp2$$Register /* tmp1 */);
388   %}
389   ins_pipe(pipe_cmpxchg);
390 %}
391 
392 instruct g1GetAndSetP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
393 %{
394   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
395   match(Set newval (GetAndSetP mem newval));
396   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
397   format %{ "xchgq    $newval, $mem" %}
398   ins_encode %{
399     assert_different_registers($mem$$Register, $newval$$Register);
400     write_barrier_pre(masm, this,
401                       $mem$$Register /* obj */,
402                       $tmp2$$Register /* pre_val */,
403                       $tmp3$$Register /* tmp */,
404                       RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
405     __ movq($tmp1$$Register, $newval$$Register);
406     __ xchgq($newval$$Register, Address($mem$$Register, 0));
407     write_barrier_post(masm, this,
408                        $mem$$Register /* store_addr */,
409                        $tmp1$$Register /* new_val */,
410                        $tmp2$$Register /* tmp1 */);
411   %}
412   ins_pipe(pipe_cmpxchg);
413 %}
414 
415 instruct g1GetAndSetN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
416 %{
417   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
418   match(Set newval (GetAndSetN mem newval));
419   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
420   format %{ "xchgq    $newval, $mem" %}
421   ins_encode %{
422     assert_different_registers($mem$$Register, $newval$$Register);
423     write_barrier_pre(masm, this,
424                       $mem$$Register /* obj */,
425                       $tmp2$$Register /* pre_val */,
426                       $tmp3$$Register /* tmp */,
427                       RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
428     __ movl($tmp1$$Register, $newval$$Register);
429     __ decode_heap_oop($tmp1$$Register);
430     __ xchgl($newval$$Register, Address($mem$$Register, 0));
431     write_barrier_post(masm, this,
432                        $mem$$Register /* store_addr */,
433                        $tmp1$$Register /* new_val */,
434                        $tmp2$$Register /* tmp1 */);
435   %}
436   ins_pipe(pipe_cmpxchg);
437 %}
438 
439 instruct g1LoadP(rRegP dst, memory mem, rRegP tmp, rFlagsReg cr)
440 %{
441   predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
442   match(Set dst (LoadP mem));
443   effect(TEMP dst, TEMP tmp, KILL cr);
444   ins_cost(125); // XXX
445   format %{ "movq    $dst, $mem\t# ptr" %}
446   ins_encode %{
447     __ movq($dst$$Register, $mem$$Address);
448     write_barrier_pre(masm, this,
449                       noreg /* obj */,
450                       $dst$$Register /* pre_val */,
451                       $tmp$$Register /* tmp */);
452   %}
453   ins_pipe(ialu_reg_mem); // XXX
454 %}
455 
456 instruct g1LoadN(rRegN dst, memory mem, rRegP tmp1, rRegP tmp2, rFlagsReg cr)
457 %{
458   predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
459   match(Set dst (LoadN mem));
460   effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
461   ins_cost(125); // XXX
462   format %{ "movl    $dst, $mem\t# compressed ptr" %}
463   ins_encode %{
464     __ movl($dst$$Register, $mem$$Address);
465     __ movl($tmp1$$Register, $dst$$Register);
466     __ decode_heap_oop($tmp1$$Register);
467     write_barrier_pre(masm, this,
468                       noreg /* obj */,
469                       $tmp1$$Register /* pre_val */,
470                       $tmp2$$Register /* tmp */);
471   %}
472   ins_pipe(ialu_reg_mem); // XXX
473 %}
--- EOF ---