1 //
  2 // Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
  3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4 //
  5 // This code is free software; you can redistribute it and/or modify it
  6 // under the terms of the GNU General Public License version 2 only, as
  7 // published by the Free Software Foundation.
  8 //
  9 // This code is distributed in the hope that it will be useful, but WITHOUT
 10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12 // version 2 for more details (a copy is included in the LICENSE file that
 13 // accompanied this code).
 14 //
 15 // You should have received a copy of the GNU General Public License version
 16 // 2 along with this work; if not, write to the Free Software Foundation,
 17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18 //
 19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20 // or visit www.oracle.com if you need additional information or have any
 21 // questions.
 22 //
 23 
 24 source_hpp %{
 25 
 26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
 27 #include "gc/shared/gc_globals.hpp"
 28 
 29 %}
 30 
 31 source %{
 32 
 33 #include "gc/g1/g1BarrierSetAssembler_x86.hpp"
 34 #include "gc/g1/g1BarrierSetRuntime.hpp"
 35 
 36 static void write_barrier_pre(MacroAssembler* masm,
 37                               const MachNode* node,
 38                               Register obj,
 39                               Register pre_val,
 40                               Register tmp,
 41                               RegSet preserve = RegSet(),
 42                               RegSet no_preserve = RegSet()) {
 43   if (!G1PreBarrierStubC2::needs_barrier(node)) {
 44     return;
 45   }
 46   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 47   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
 48   G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
 49   for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
 50     stub->preserve(*reg);
 51   }
 52   for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
 53     stub->dont_preserve(*reg);
 54   }
 55   g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, r15_thread, tmp, stub);
 56 }
 57 
 58 static void write_barrier_post(MacroAssembler* masm,
 59                                const MachNode* node,
 60                                Register store_addr,
 61                                Register new_val,
 62                                Register tmp1,
 63                                Register tmp2) {
 64   if (!G1PostBarrierStubC2::needs_barrier(node)) {
 65     return;
 66   }
 67   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 68   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
 69   G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
 70   g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, r15_thread, tmp1, tmp2, stub);
 71 }
 72 
 73 %}
 74 
 75 instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
 76 %{
 77   predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
 78   match(Set mem (StoreP mem src));
 79   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
 80   ins_cost(125); // XXX
 81   format %{ "movq    $mem, $src\t# ptr" %}
 82   ins_encode %{
 83     // Materialize the store address internally (as opposed to defining 'mem' as
 84     // an indirect memory operand) to reduce the overhead of LCM when processing
 85     // large basic blocks with many stores. Such basic blocks arise, for
 86     // instance, from static initializations of large String arrays.
 87     // The same holds for g1StoreN and g1EncodePAndStoreN.
 88     __ lea($tmp1$$Register, $mem$$Address);
 89     write_barrier_pre(masm, this,
 90                       $tmp1$$Register /* obj */,
 91                       $tmp2$$Register /* pre_val */,
 92                       $tmp3$$Register /* tmp */,
 93                       RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
 94     __ movq(Address($tmp1$$Register, 0), $src$$Register);
 95     write_barrier_post(masm, this,
 96                        $tmp1$$Register /* store_addr */,
 97                        $src$$Register /* new_val */,
 98                        $tmp3$$Register /* tmp1 */,
 99                        $tmp2$$Register /* tmp2 */);
100   %}
101   ins_pipe(ialu_mem_reg);
102 %}
103 
104 instruct g1StoreN(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
105 %{
106   predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
107   match(Set mem (StoreN mem src));
108   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
109   ins_cost(125); // XXX
110   format %{ "movl    $mem, $src\t# ptr" %}
111   ins_encode %{
112     __ lea($tmp1$$Register, $mem$$Address);
113     write_barrier_pre(masm, this,
114                       $tmp1$$Register /* obj */,
115                       $tmp2$$Register /* pre_val */,
116                       $tmp3$$Register /* tmp */,
117                       RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
118     __ movl(Address($tmp1$$Register, 0), $src$$Register);
119     if ((barrier_data() & G1C2BarrierPost) != 0) {
120       __ movl($tmp2$$Register, $src$$Register);
121       if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
122         __ decode_heap_oop($tmp2$$Register);
123       } else {
124         __ decode_heap_oop_not_null($tmp2$$Register);
125       }
126     }
127     write_barrier_post(masm, this,
128                        $tmp1$$Register /* store_addr */,
129                        $tmp2$$Register /* new_val */,
130                        $tmp3$$Register /* tmp1 */,
131                        $tmp2$$Register /* tmp2 */);
132   %}
133   ins_pipe(ialu_mem_reg);
134 %}
135 
136 instruct g1EncodePAndStoreN(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
137 %{
138   predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
139   match(Set mem (StoreN mem (EncodeP src)));
140   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
141   ins_cost(125); // XXX
142   format %{ "encode_heap_oop $src\n\t"
143             "movl   $mem, $src\t# ptr" %}
144   ins_encode %{
145     __ lea($tmp1$$Register, $mem$$Address);
146     write_barrier_pre(masm, this,
147                       $tmp1$$Register /* obj */,
148                       $tmp2$$Register /* pre_val */,
149                       $tmp3$$Register /* tmp */,
150                       RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
151     __ movq($tmp2$$Register, $src$$Register);
152     if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
153       __ encode_heap_oop($tmp2$$Register);
154     } else {
155       __ encode_heap_oop_not_null($tmp2$$Register);
156     }
157     __ movl(Address($tmp1$$Register, 0), $tmp2$$Register);
158     write_barrier_post(masm, this,
159                        $tmp1$$Register /* store_addr */,
160                        $src$$Register /* new_val */,
161                        $tmp3$$Register /* tmp1 */,
162                        $tmp2$$Register /* tmp2 */);
163   %}
164   ins_pipe(ialu_mem_reg);
165 %}
166 
167 instruct g1CompareAndExchangeP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
168 %{
169   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
170   match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
171   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
172   format %{ "lock\n\t"
173             "cmpxchgq $newval, $mem" %}
174   ins_encode %{
175     assert_different_registers($oldval$$Register, $mem$$Register);
176     // Pass $oldval to the pre-barrier (instead of loading from $mem), because
177     // $oldval is the only value that can be overwritten.
178     // The same holds for g1CompareAndSwapP.
179     write_barrier_pre(masm, this,
180                       noreg /* obj */,
181                       $oldval$$Register /* pre_val */,
182                       $tmp3$$Register /* tmp */,
183                       RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
184     __ movq($tmp1$$Register, $newval$$Register);
185     __ lock();
186     __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
187     write_barrier_post(masm, this,
188                        $mem$$Register /* store_addr */,
189                        $tmp1$$Register /* new_val */,
190                        $tmp2$$Register /* tmp1 */,
191                        $tmp3$$Register /* tmp2 */);
192   %}
193   ins_pipe(pipe_cmpxchg);
194 %}
195 
196 instruct g1CompareAndExchangeN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
197 %{
198   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
199   match(Set oldval (CompareAndExchangeN mem (Binary oldval newval)));
200   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
201   format %{ "lock\n\t"
202             "cmpxchgq $newval, $mem" %}
203   ins_encode %{
204     assert_different_registers($oldval$$Register, $mem$$Register);
205     write_barrier_pre(masm, this,
206                       $mem$$Register /* obj */,
207                       $tmp2$$Register /* pre_val */,
208                       $tmp3$$Register /* tmp */,
209                       RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
210     __ movl($tmp1$$Register, $newval$$Register);
211     __ lock();
212     __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
213     __ decode_heap_oop($tmp1$$Register);
214     write_barrier_post(masm, this,
215                        $mem$$Register /* store_addr */,
216                        $tmp1$$Register /* new_val */,
217                        $tmp2$$Register /* tmp1 */,
218                        $tmp3$$Register /* tmp2 */);
219   %}
220   ins_pipe(pipe_cmpxchg);
221 %}
222 
223 instruct g1CompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
224 %{
225   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
226   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
227   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
228   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
229   format %{ "lock\n\t"
230             "cmpxchgq $newval, $mem\n\t"
231             "sete     $res\n\t"
232             "movzbl   $res, $res" %}
233   ins_encode %{
234     assert_different_registers($oldval$$Register, $mem$$Register);
235     write_barrier_pre(masm, this,
236                       noreg /* obj */,
237                       $oldval$$Register /* pre_val */,
238                       $tmp3$$Register /* tmp */,
239                       RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
240                       RegSet::of($res$$Register) /* no_preserve */);
241     __ movq($tmp1$$Register, $newval$$Register);
242     __ lock();
243     __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
244     __ setb(Assembler::equal, $res$$Register);
245     __ movzbl($res$$Register, $res$$Register);
246     write_barrier_post(masm, this,
247                        $mem$$Register /* store_addr */,
248                        $tmp1$$Register /* new_val */,
249                        $tmp2$$Register /* tmp1 */,
250                        $tmp3$$Register /* tmp2 */);
251   %}
252   ins_pipe(pipe_cmpxchg);
253 %}
254 
255 instruct g1CompareAndSwapN(rRegI res, indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
256 %{
257   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
258   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
259   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
260   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
261   format %{ "lock\n\t"
262             "cmpxchgq $newval, $mem\n\t"
263             "sete     $res\n\t"
264             "movzbl   $res, $res" %}
265   ins_encode %{
266     assert_different_registers($oldval$$Register, $mem$$Register);
267     write_barrier_pre(masm, this,
268                       $mem$$Register /* obj */,
269                       $tmp2$$Register /* pre_val */,
270                       $tmp3$$Register /* tmp */,
271                       RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
272                       RegSet::of($res$$Register) /* no_preserve */);
273     __ movl($tmp1$$Register, $newval$$Register);
274     __ lock();
275     __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
276     __ setb(Assembler::equal, $res$$Register);
277     __ movzbl($res$$Register, $res$$Register);
278     __ decode_heap_oop($tmp1$$Register);
279     write_barrier_post(masm, this,
280                        $mem$$Register /* store_addr */,
281                        $tmp1$$Register /* new_val */,
282                        $tmp2$$Register /* tmp1 */,
283                        $tmp3$$Register /* tmp2 */);
284   %}
285   ins_pipe(pipe_cmpxchg);
286 %}
287 
288 instruct g1GetAndSetP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
289 %{
290   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
291   match(Set newval (GetAndSetP mem newval));
292   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
293   format %{ "xchgq    $newval, $mem" %}
294   ins_encode %{
295     assert_different_registers($mem$$Register, $newval$$Register);
296     write_barrier_pre(masm, this,
297                       $mem$$Register /* obj */,
298                       $tmp2$$Register /* pre_val */,
299                       $tmp3$$Register /* tmp */,
300                       RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
301     __ movq($tmp1$$Register, $newval$$Register);
302     __ xchgq($newval$$Register, Address($mem$$Register, 0));
303     write_barrier_post(masm, this,
304                        $mem$$Register /* store_addr */,
305                        $tmp1$$Register /* new_val */,
306                        $tmp2$$Register /* tmp1 */,
307                        $tmp3$$Register /* tmp2 */);
308   %}
309   ins_pipe(pipe_cmpxchg);
310 %}
311 
312 instruct g1GetAndSetN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
313 %{
314   predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
315   match(Set newval (GetAndSetN mem newval));
316   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
317   format %{ "xchgq    $newval, $mem" %}
318   ins_encode %{
319     assert_different_registers($mem$$Register, $newval$$Register);
320     write_barrier_pre(masm, this,
321                       $mem$$Register /* obj */,
322                       $tmp2$$Register /* pre_val */,
323                       $tmp3$$Register /* tmp */,
324                       RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
325     __ movl($tmp1$$Register, $newval$$Register);
326     __ decode_heap_oop($tmp1$$Register);
327     __ xchgl($newval$$Register, Address($mem$$Register, 0));
328     write_barrier_post(masm, this,
329                        $mem$$Register /* store_addr */,
330                        $tmp1$$Register /* new_val */,
331                        $tmp2$$Register /* tmp1 */,
332                        $tmp3$$Register /* tmp2 */);
333   %}
334   ins_pipe(pipe_cmpxchg);
335 %}
336 
337 instruct g1LoadP(rRegP dst, memory mem, rRegP tmp, rFlagsReg cr)
338 %{
339   predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
340   match(Set dst (LoadP mem));
341   effect(TEMP dst, TEMP tmp, KILL cr);
342   ins_cost(125); // XXX
343   format %{ "movq    $dst, $mem\t# ptr" %}
344   ins_encode %{
345     __ movq($dst$$Register, $mem$$Address);
346     write_barrier_pre(masm, this,
347                       noreg /* obj */,
348                       $dst$$Register /* pre_val */,
349                       $tmp$$Register /* tmp */);
350   %}
351   ins_pipe(ialu_reg_mem); // XXX
352 %}
353 
354 instruct g1LoadN(rRegN dst, memory mem, rRegP tmp1, rRegP tmp2, rFlagsReg cr)
355 %{
356   predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
357   match(Set dst (LoadN mem));
358   effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
359   ins_cost(125); // XXX
360   format %{ "movl    $dst, $mem\t# compressed ptr" %}
361   ins_encode %{
362     __ movl($dst$$Register, $mem$$Address);
363     __ movl($tmp1$$Register, $dst$$Register);
364     __ decode_heap_oop($tmp1$$Register);
365     write_barrier_pre(masm, this,
366                       noreg /* obj */,
367                       $tmp1$$Register /* pre_val */,
368                       $tmp2$$Register /* tmp */);
369   %}
370   ins_pipe(ialu_reg_mem); // XXX
371 %}