1 /*
  2  * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "asm/macroAssembler.inline.hpp"
 26 #include "gc/g1/g1BarrierSet.hpp"
 27 #include "gc/g1/g1BarrierSetAssembler.hpp"
 28 #include "gc/g1/g1BarrierSetRuntime.hpp"
 29 #include "gc/g1/g1CardTable.hpp"
 30 #include "gc/g1/g1HeapRegion.hpp"
 31 #include "gc/g1/g1ThreadLocalData.hpp"
 32 #include "interpreter/interp_masm.hpp"
 33 #include "runtime/sharedRuntime.hpp"
 34 #include "utilities/debug.hpp"
 35 #include "utilities/macros.hpp"
 36 #ifdef COMPILER1
 37 #include "c1/c1_LIRAssembler.hpp"
 38 #include "c1/c1_MacroAssembler.hpp"
 39 #include "gc/g1/c1/g1BarrierSetC1.hpp"
 40 #endif // COMPILER1
 41 #ifdef COMPILER2
 42 #include "gc/g1/c2/g1BarrierSetC2.hpp"
 43 #endif // COMPILER2
 44 
 45 #define __ masm->
 46 
 47 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
 48                                                             Register addr, Register count) {
 49   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 50 
 51   if (!dest_uninitialized) {
 52     Register thread = r15_thread;
 53 
 54     Label filtered;
 55     Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
 56     // Is marking active?
 57     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 58       __ cmpl(in_progress, 0);
 59     } else {
 60       assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 61       __ cmpb(in_progress, 0);
 62     }
 63 
 64     __ jcc(Assembler::equal, filtered);
 65 
 66     __ push_call_clobbered_registers(false /* save_fpu */);
 67     if (count == c_rarg0) {
 68       if (addr == c_rarg1) {
 69         // exactly backwards!!
 70         __ xchgptr(c_rarg1, c_rarg0);
 71       } else {
 72         __ movptr(c_rarg1, count);
 73         __ movptr(c_rarg0, addr);
 74       }
 75     } else {
 76       __ movptr(c_rarg0, addr);
 77       __ movptr(c_rarg1, count);
 78     }
 79     if (UseCompressedOops) {
 80       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
 81     } else {
 82       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
 83     }
 84     __ pop_call_clobbered_registers(false /* save_fpu */);
 85 
 86     __ bind(filtered);
 87   }
 88 }
 89 
 90 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 91                                                              Register addr, Register count, Register tmp) {
 92   Label done;
 93 
 94   __ testptr(count, count);
 95   __ jcc(Assembler::zero, done);
 96 
 97   // Calculate end address in "count".
 98   Address::ScaleFactor scale = UseCompressedOops ? Address::times_4 : Address::times_8;
 99   __ leaq(count, Address(addr, count, scale));
100 
101   // Calculate start card address in "addr".
102   __ shrptr(addr, CardTable::card_shift());
103 
104   Register thread = r15_thread;
105 
106   __ movptr(tmp, Address(thread, in_bytes(G1ThreadLocalData::card_table_base_offset())));
107   __ addptr(addr, tmp);
108 
109   // Calculate address of card of last word in the array.
110   __ subptr(count, 1);
111   __ shrptr(count, CardTable::card_shift());
112   __ addptr(count, tmp);
113 
114   Label loop;
115   // Iterate from start card to end card (inclusive).
116   __ bind(loop);
117 
118   Label is_clean_card;
119   if (UseCondCardMark) {
120     __ cmpb(Address(addr, 0), G1CardTable::clean_card_val());
121     __ jcc(Assembler::equal, is_clean_card);
122   } else {
123    __ movb(Address(addr, 0), G1CardTable::dirty_card_val());
124   }
125 
126   Label next_card;
127   __ bind(next_card);
128   __ addptr(addr, sizeof(CardTable::CardValue));
129   __ cmpptr(addr, count);
130   __ jcc(Assembler::belowEqual, loop);
131   __ jmp(done);
132 
133   __ bind(is_clean_card);
134   // Card was clean. Dirty card and go to next..
135   __ movb(Address(addr, 0), G1CardTable::dirty_card_val());
136   __ jmp(next_card);
137 
138   __ bind(done);
139 }
140 
141 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
142                                     Register dst, Address src, Register tmp1) {
143   bool on_oop = is_reference_type(type);
144   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
145   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
146   bool on_reference = on_weak || on_phantom;
147   ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1);
148   if (on_oop && on_reference) {
149     // Generate the G1 pre-barrier code to log the value of
150     // the referent field in an SATB buffer.
151     g1_write_barrier_pre(masm /* masm */,
152                          noreg /* obj */,
153                          dst /* pre_val */,
154                          tmp1 /* tmp */,
155                          true /* tosca_live */,
156                          true /* expand_call */);
157   }
158 }
159 
160 static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
161                                      const Register thread, const Register value, const Register temp) {
162   // This code assumes that buffer index is pointer sized.
163   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
164   // Can we store a value in the given thread's buffer?
165   // (The index field is typed as size_t.)
166   __ movptr(temp, Address(thread, in_bytes(index_offset)));   // temp := *(index address)
167   __ testptr(temp, temp);                                     // index == 0?
168   __ jcc(Assembler::zero, runtime);                           // jump to runtime if index == 0 (full buffer)
169   // The buffer is not full, store value into it.
170   __ subptr(temp, wordSize);                                  // temp := next index
171   __ movptr(Address(thread, in_bytes(index_offset)), temp);   // *(index address) := next index
172   __ addptr(temp, Address(thread, in_bytes(buffer_offset)));  // temp := buffer address + next index
173   __ movptr(Address(temp, 0), value);                         // *(buffer address + next index) := value
174 }
175 
176 static void generate_pre_barrier_fast_path(MacroAssembler* masm,
177                                            const Register thread) {
178   Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
179   // Is marking active?
180   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
181     __ cmpl(in_progress, 0);
182   } else {
183     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
184     __ cmpb(in_progress, 0);
185   }
186 }
187 
188 static void generate_pre_barrier_slow_path(MacroAssembler* masm,
189                                            const Register obj,
190                                            const Register pre_val,
191                                            const Register thread,
192                                            const Register tmp,
193                                            Label& done,
194                                            Label& runtime) {
195   // Do we need to load the previous value?
196   if (obj != noreg) {
197     __ load_heap_oop(pre_val, Address(obj, 0), noreg, AS_RAW);
198   }
199   // Is the previous value null?
200   __ cmpptr(pre_val, NULL_WORD);
201   __ jcc(Assembler::equal, done);
202   generate_queue_insertion(masm,
203                            G1ThreadLocalData::satb_mark_queue_index_offset(),
204                            G1ThreadLocalData::satb_mark_queue_buffer_offset(),
205                            runtime,
206                            thread, pre_val, tmp);
207   __ jmp(done);
208 }
209 
210 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
211                                                  Register obj,
212                                                  Register pre_val,
213                                                  Register tmp,
214                                                  bool tosca_live,
215                                                  bool expand_call) {
216   // If expand_call is true then we expand the call_VM_leaf macro
217   // directly to skip generating the check by
218   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
219   const Register thread = r15_thread;
220 
221   Label done;
222   Label runtime;
223 
224   assert(pre_val != noreg, "check this code");
225 
226   if (obj != noreg) {
227     assert_different_registers(obj, pre_val, tmp);
228     assert(pre_val != rax, "check this code");
229   }
230 
231   generate_pre_barrier_fast_path(masm, thread);
232   // If marking is not active (*(mark queue active address) == 0), jump to done
233   __ jcc(Assembler::equal, done);
234   generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
235 
236   __ bind(runtime);
237 
238   // Determine and save the live input values
239   __ push_call_clobbered_registers();
240 
241   // Calling the runtime using the regular call_VM_leaf mechanism generates
242   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
243   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
244   //
245   // If we care generating the pre-barrier without a frame (e.g. in the
246   // intrinsified Reference.get() routine) then ebp might be pointing to
247   // the caller frame and so this check will most likely fail at runtime.
248   //
249   // Expanding the call directly bypasses the generation of the check.
250   // So when we do not have have a full interpreter frame on the stack
251   // expand_call should be passed true.
252 
253   if (expand_call) {
254     assert(pre_val != c_rarg1, "smashed arg");
255     if (c_rarg1 != thread) {
256       __ mov(c_rarg1, thread);
257     }
258     if (c_rarg0 != pre_val) {
259       __ mov(c_rarg0, pre_val);
260     }
261     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
262   } else {
263     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
264   }
265 
266   __ pop_call_clobbered_registers();
267 
268   __ bind(done);
269 }
270 
271 static void generate_post_barrier_fast_path(MacroAssembler* masm,
272                                             const Register store_addr,
273                                             const Register new_val,
274                                             const Register tmp1,
275                                             Label& done,
276                                             bool new_val_may_be_null) {
277 
278   assert_different_registers(store_addr, new_val, tmp1, noreg);
279 
280   Register thread = r15_thread;
281 
282   // Does store cross heap regions?
283   __ movptr(tmp1, store_addr);                                    // tmp1 := store address
284   __ xorptr(tmp1, new_val);                                       // tmp1 := store address ^ new value
285   __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes);               // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
286   __ jcc(Assembler::equal, done);
287 
288   // Crosses regions, storing null?
289   if (new_val_may_be_null) {
290     __ cmpptr(new_val, NULL_WORD);                                // new value == null?
291     __ jcc(Assembler::equal, done);
292   }
293 
294   __ movptr(tmp1, store_addr);                                    // tmp1 := store address
295   __ shrptr(tmp1, CardTable::card_shift());                       // tmp1 := card address relative to card table base
296 
297   Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()));
298   __ addptr(tmp1, card_table_addr);                               // tmp1 := card address
299   if (UseCondCardMark) {
300     __ cmpb(Address(tmp1, 0), G1CardTable::clean_card_val());     // *(card address) == clean_card_val?
301     __ jcc(Assembler::notEqual, done);
302   }
303   // Storing a region crossing, non-null oop, card is clean.
304   // Dirty card.
305   __ movb(Address(tmp1, 0), G1CardTable::dirty_card_val());       // *(card address) := dirty_card_val
306 }
307 
308 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
309                                                   Register store_addr,
310                                                   Register new_val,
311                                                   Register tmp) {
312   Label done;
313   generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, done, true /* new_val_may_be_null */);
314   __ bind(done);
315 }
316 
317 #if defined(COMPILER2)
318 
319 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
320   SaveLiveRegisters save_registers(masm, stub);
321   if (c_rarg0 != arg) {
322     __ mov(c_rarg0, arg);
323   }
324   __ mov(c_rarg1, r15_thread);
325   // rax is a caller-saved, non-argument-passing register, so it does not
326   // interfere with c_rarg0 or c_rarg1. If it contained any live value before
327   // entering this stub, it is saved at this point, and restored after the
328   // call. If it did not contain any live value, it is free to be used. In
329   // either case, it is safe to use it here as a call scratch register.
330   __ call(RuntimeAddress(runtime_path), rax);
331 }
332 
333 void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
334                                                     Register obj,
335                                                     Register pre_val,
336                                                     Register tmp,
337                                                     G1PreBarrierStubC2* stub) {
338   const Register thread = r15_thread;
339 
340   assert(pre_val != noreg, "check this code");
341   if (obj != noreg) {
342     assert_different_registers(obj, pre_val, tmp);
343   }
344 
345   stub->initialize_registers(obj, pre_val, thread, tmp);
346 
347   generate_pre_barrier_fast_path(masm, thread);
348   // If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
349   __ jcc(Assembler::notEqual, *stub->entry());
350 
351   __ bind(*stub->continuation());
352 }
353 
354 void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
355                                                          G1PreBarrierStubC2* stub) const {
356   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
357   Label runtime;
358   Register obj = stub->obj();
359   Register pre_val = stub->pre_val();
360   Register thread = stub->thread();
361   Register tmp = stub->tmp1();
362   assert(stub->tmp2() == noreg, "not needed in this platform");
363 
364   __ bind(*stub->entry());
365   generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, *stub->continuation(), runtime);
366 
367   __ bind(runtime);
368   generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
369   __ jmp(*stub->continuation());
370 }
371 
372 void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
373                                                      Register store_addr,
374                                                      Register new_val,
375                                                      Register tmp,
376                                                      bool new_val_may_be_null) {
377   Label done;
378   generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, done, new_val_may_be_null);
379   __ bind(done);
380 }
381 
382 #endif // COMPILER2
383 
384 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
385                                          Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
386   bool in_heap = (decorators & IN_HEAP) != 0;
387   bool as_normal = (decorators & AS_NORMAL) != 0;
388 
389   bool needs_pre_barrier = as_normal;
390   bool needs_post_barrier = val != noreg && in_heap;
391 
392   // flatten object address if needed
393   // We do it regardless of precise because we need the registers
394   if (dst.index() == noreg && dst.disp() == 0) {
395     if (dst.base() != tmp1) {
396       __ movptr(tmp1, dst.base());
397     }
398   } else {
399     __ lea(tmp1, dst);
400   }
401 
402   if (needs_pre_barrier) {
403     g1_write_barrier_pre(masm /*masm*/,
404                          tmp1 /* obj */,
405                          tmp2 /* pre_val */,
406                          tmp3  /* tmp */,
407                          val != noreg /* tosca_live */,
408                          false /* expand_call */);
409   }
410   if (val == noreg) {
411     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
412   } else {
413     Register new_val = val;
414     if (needs_post_barrier) {
415       // G1 barrier needs uncompressed oop for region cross check.
416       if (UseCompressedOops) {
417         new_val = tmp2;
418         __ movptr(new_val, val);
419       }
420     }
421     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
422     if (needs_post_barrier) {
423       g1_write_barrier_post(masm /*masm*/,
424                             tmp1 /* store_adr */,
425                             new_val /* new_val */,
426                             tmp3 /* tmp */);
427     }
428   }
429 }
430 
431 #ifdef COMPILER1
432 
433 #undef __
434 #define __ ce->masm()->
435 
436 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
437   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
438   // At this point we know that marking is in progress.
439   // If do_load() is true then we have to emit the
440   // load of the previous value; otherwise it has already
441   // been loaded into _pre_val.
442 
443   __ bind(*stub->entry());
444   assert(stub->pre_val()->is_register(), "Precondition.");
445 
446   Register pre_val_reg = stub->pre_val()->as_register();
447 
448   if (stub->do_load()) {
449     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
450   }
451 
452   __ cmpptr(pre_val_reg, NULL_WORD);
453   __ jcc(Assembler::equal, *stub->continuation());
454   ce->store_parameter(stub->pre_val()->as_register(), 0);
455   __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
456   __ jmp(*stub->continuation());
457 
458 }
459 
460 #undef __
461 
462 void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
463                                                      Register store_addr,
464                                                      Register new_val,
465                                                      Register thread,
466                                                      Register tmp1,
467                                                      Register tmp2 /* unused on x86 */) {
468   Label done;
469   generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, done, true /* new_val_may_be_null */);
470   masm->bind(done);
471 }
472 
473 #define __ sasm->
474 
475 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
476   // Generated code assumes that buffer index is pointer sized.
477   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
478 
479   __ prologue("g1_pre_barrier", false);
480   // arg0 : previous value of memory
481 
482   __ push_ppx(rax);
483   __ push_ppx(rdx);
484 
485   const Register pre_val = rax;
486   const Register thread = r15_thread;
487   const Register tmp = rdx;
488 
489   Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
490   Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
491   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
492 
493   Label done;
494   Label runtime;
495 
496   // Is marking still active?
497   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
498     __ cmpl(queue_active, 0);
499   } else {
500     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
501     __ cmpb(queue_active, 0);
502   }
503   __ jcc(Assembler::equal, done);
504 
505   // Can we store original value in the thread's buffer?
506 
507   __ movptr(tmp, queue_index);
508   __ testptr(tmp, tmp);
509   __ jcc(Assembler::zero, runtime);
510   __ subptr(tmp, wordSize);
511   __ movptr(queue_index, tmp);
512   __ addptr(tmp, buffer);
513 
514   // prev_val (rax)
515   __ load_parameter(0, pre_val);
516   __ movptr(Address(tmp, 0), pre_val);
517   __ jmp(done);
518 
519   __ bind(runtime);
520 
521   __ push_call_clobbered_registers();
522 
523   // load the pre-value
524   __ load_parameter(0, rcx);
525   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread);
526 
527   __ pop_call_clobbered_registers();
528 
529   __ bind(done);
530 
531   __ pop_ppx(rdx);
532   __ pop_ppx(rax);
533 
534   __ epilogue();
535 }
536 
537 #undef __
538 
539 #endif // COMPILER1