1 /*
  2  * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "asm/macroAssembler.inline.hpp"
 26 #include "code/aotCodeCache.hpp"
 27 #include "gc/g1/g1BarrierSet.hpp"
 28 #include "gc/g1/g1BarrierSetAssembler.hpp"
 29 #include "gc/g1/g1BarrierSetRuntime.hpp"
 30 #include "gc/g1/g1CardTable.hpp"
 31 #include "gc/g1/g1HeapRegion.hpp"
 32 #include "gc/g1/g1ThreadLocalData.hpp"
 33 #include "interpreter/interp_masm.hpp"
 34 #include "runtime/sharedRuntime.hpp"
 35 #include "utilities/debug.hpp"
 36 #include "utilities/macros.hpp"
 37 #ifdef COMPILER1
 38 #include "c1/c1_LIRAssembler.hpp"
 39 #include "c1/c1_MacroAssembler.hpp"
 40 #include "gc/g1/c1/g1BarrierSetC1.hpp"
 41 #endif // COMPILER1
 42 #ifdef COMPILER2
 43 #include "gc/g1/c2/g1BarrierSetC2.hpp"
 44 #endif // COMPILER2
 45 
 46 #define __ masm->
 47 
 48 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
 49                                                             Register addr, Register count) {
 50   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 51 
 52   if (!dest_uninitialized) {
 53     Register thread = r15_thread;
 54 
 55     Label filtered;
 56     Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
 57     // Is marking active?
 58     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 59       __ cmpl(in_progress, 0);
 60     } else {
 61       assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 62       __ cmpb(in_progress, 0);
 63     }
 64 
 65     __ jcc(Assembler::equal, filtered);
 66 
 67     __ push_call_clobbered_registers(false /* save_fpu */);
 68     if (count == c_rarg0) {
 69       if (addr == c_rarg1) {
 70         // exactly backwards!!
 71         __ xchgptr(c_rarg1, c_rarg0);
 72       } else {
 73         __ movptr(c_rarg1, count);
 74         __ movptr(c_rarg0, addr);
 75       }
 76     } else {
 77       __ movptr(c_rarg0, addr);
 78       __ movptr(c_rarg1, count);
 79     }
 80     if (UseCompressedOops) {
 81       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
 82     } else {
 83       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
 84     }
 85     __ pop_call_clobbered_registers(false /* save_fpu */);
 86 
 87     __ bind(filtered);
 88   }
 89 }
 90 
 91 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 92                                                              Register addr, Register count, Register tmp) {
 93   Label done;
 94 
 95   __ testptr(count, count);
 96   __ jcc(Assembler::zero, done);
 97 
 98   // Calculate end address in "count".
 99   Address::ScaleFactor scale = UseCompressedOops ? Address::times_4 : Address::times_8;
100   __ leaq(count, Address(addr, count, scale));
101 
102   // Calculate start card address in "addr".
103   __ shrptr(addr, CardTable::card_shift());
104 
105   Register thread = r15_thread;
106 
107   __ movptr(tmp, Address(thread, in_bytes(G1ThreadLocalData::card_table_base_offset())));
108   __ addptr(addr, tmp);
109 
110   // Calculate address of card of last word in the array.
111   __ subptr(count, 1);
112   __ shrptr(count, CardTable::card_shift());
113   __ addptr(count, tmp);
114 
115   Label loop;
116   // Iterate from start card to end card (inclusive).
117   __ bind(loop);
118 
119   Label is_clean_card;
120   if (UseCondCardMark) {
121     __ cmpb(Address(addr, 0), G1CardTable::clean_card_val());
122     __ jcc(Assembler::equal, is_clean_card);
123   } else {
124    __ movb(Address(addr, 0), G1CardTable::dirty_card_val());
125   }
126 
127   Label next_card;
128   __ bind(next_card);
129   __ addptr(addr, sizeof(CardTable::CardValue));
130   __ cmpptr(addr, count);
131   __ jcc(Assembler::belowEqual, loop);
132   __ jmp(done);
133 
134   __ bind(is_clean_card);
135   // Card was clean. Dirty card and go to next..
136   __ movb(Address(addr, 0), G1CardTable::dirty_card_val());
137   __ jmp(next_card);
138 
139   __ bind(done);
140 }
141 
142 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
143                                     Register dst, Address src, Register tmp1) {
144   bool on_oop = is_reference_type(type);
145   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
146   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
147   bool on_reference = on_weak || on_phantom;
148   ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1);
149   if (on_oop && on_reference) {
150     // Generate the G1 pre-barrier code to log the value of
151     // the referent field in an SATB buffer.
152     g1_write_barrier_pre(masm /* masm */,
153                          noreg /* obj */,
154                          dst /* pre_val */,
155                          tmp1 /* tmp */,
156                          true /* tosca_live */,
157                          true /* expand_call */);
158   }
159 }
160 
161 static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
162                                      const Register thread, const Register value, const Register temp) {
163   // This code assumes that buffer index is pointer sized.
164   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
165   // Can we store a value in the given thread's buffer?
166   // (The index field is typed as size_t.)
167   __ movptr(temp, Address(thread, in_bytes(index_offset)));   // temp := *(index address)
168   __ testptr(temp, temp);                                     // index == 0?
169   __ jcc(Assembler::zero, runtime);                           // jump to runtime if index == 0 (full buffer)
170   // The buffer is not full, store value into it.
171   __ subptr(temp, wordSize);                                  // temp := next index
172   __ movptr(Address(thread, in_bytes(index_offset)), temp);   // *(index address) := next index
173   __ addptr(temp, Address(thread, in_bytes(buffer_offset)));  // temp := buffer address + next index
174   __ movptr(Address(temp, 0), value);                         // *(buffer address + next index) := value
175 }
176 
177 static void generate_pre_barrier_fast_path(MacroAssembler* masm,
178                                            const Register thread) {
179   Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
180   // Is marking active?
181   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
182     __ cmpl(in_progress, 0);
183   } else {
184     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
185     __ cmpb(in_progress, 0);
186   }
187 }
188 
189 static void generate_pre_barrier_slow_path(MacroAssembler* masm,
190                                            const Register obj,
191                                            const Register pre_val,
192                                            const Register thread,
193                                            const Register tmp,
194                                            Label& done,
195                                            Label& runtime) {
196   // Do we need to load the previous value?
197   if (obj != noreg) {
198     __ load_heap_oop(pre_val, Address(obj, 0), noreg, AS_RAW);
199   }
200   // Is the previous value null?
201   __ cmpptr(pre_val, NULL_WORD);
202   __ jcc(Assembler::equal, done);
203   generate_queue_insertion(masm,
204                            G1ThreadLocalData::satb_mark_queue_index_offset(),
205                            G1ThreadLocalData::satb_mark_queue_buffer_offset(),
206                            runtime,
207                            thread, pre_val, tmp);
208   __ jmp(done);
209 }
210 
211 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
212                                                  Register obj,
213                                                  Register pre_val,
214                                                  Register tmp,
215                                                  bool tosca_live,
216                                                  bool expand_call) {
217   // If expand_call is true then we expand the call_VM_leaf macro
218   // directly to skip generating the check by
219   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
220   const Register thread = r15_thread;
221 
222   Label done;
223   Label runtime;
224 
225   assert(pre_val != noreg, "check this code");
226 
227   if (obj != noreg) {
228     assert_different_registers(obj, pre_val, tmp);
229     assert(pre_val != rax, "check this code");
230   }
231 
232   generate_pre_barrier_fast_path(masm, thread);
233   // If marking is not active (*(mark queue active address) == 0), jump to done
234   __ jcc(Assembler::equal, done);
235   generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
236 
237   __ bind(runtime);
238 
239   // Determine and save the live input values
240   __ push_call_clobbered_registers();
241 
242   // Calling the runtime using the regular call_VM_leaf mechanism generates
243   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
244   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
245   //
246   // If we care generating the pre-barrier without a frame (e.g. in the
247   // intrinsified Reference.get() routine) then ebp might be pointing to
248   // the caller frame and so this check will most likely fail at runtime.
249   //
250   // Expanding the call directly bypasses the generation of the check.
251   // So when we do not have have a full interpreter frame on the stack
252   // expand_call should be passed true.
253 
254   if (expand_call) {
255     assert(pre_val != c_rarg1, "smashed arg");
256     if (c_rarg1 != thread) {
257       __ mov(c_rarg1, thread);
258     }
259     if (c_rarg0 != pre_val) {
260       __ mov(c_rarg0, pre_val);
261     }
262     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
263   } else {
264     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
265   }
266 
267   __ pop_call_clobbered_registers();
268 
269   __ bind(done);
270 }
271 
272 // return a register that differs from reg1, reg2, reg3 and is not rcx
273 
274 static Register pick_different_reg(Register reg1, Register reg2 = noreg, Register reg3= noreg, Register reg4 = noreg) {
275   RegSet available = (RegSet::of(rscratch1, rscratch2, rax, rbx) + rdx -
276                       RegSet::of(reg1, reg2, reg3, reg4));
277   return *(available.begin());
278 }
279 
280 static void generate_post_barrier_fast_path(MacroAssembler* masm,
281                                             const Register store_addr,
282                                             const Register new_val,
283                                             const Register tmp1,
284                                             Label& done,
285                                             bool new_val_may_be_null) {
286 
287   assert_different_registers(store_addr, new_val, tmp1, noreg);
288 
289   Register thread = r15_thread;
290 
291   // Does store cross heap regions?
292 #if INCLUDE_CDS
293   // AOT code needs to load the barrier grain shift from the aot
294   // runtime constants area in the code cache otherwise we can compile
295   // it as an immediate operand
296 
297   if (AOTCodeCache::is_on_for_dump()) {
298     address grain_shift_addr = AOTRuntimeConstants::grain_shift_address();
299     Register save = pick_different_reg(rcx, tmp1, new_val, store_addr);
300     __ push(save);
301     __ movptr(save, store_addr);
302     __ xorptr(save, new_val);
303     __ push(rcx);
304     __ lea(rcx, ExternalAddress(grain_shift_addr));
305     __ movptr(rcx, Address(rcx, 0));
306     __ shrptr(save);
307     __ pop(rcx);
308     __ mov(tmp1, save);
309     __ pop(save);
310     __ jcc(Assembler::equal, done);
311   } else
312 #endif // INCLUDE_CDS
313   {
314     __ movptr(tmp1, store_addr);                                  // tmp1 := store address
315     __ xorptr(tmp1, new_val);                                     // tmp1 := store address ^ new value
316     __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes);             // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
317     __ jcc(Assembler::equal, done);
318   }
319 
320   // Crosses regions, storing null?
321   if (new_val_may_be_null) {
322     __ cmpptr(new_val, NULL_WORD);                                // new value == null?
323     __ jcc(Assembler::equal, done);
324   }
325 
326   __ movptr(tmp1, store_addr);                                    // tmp1 := store address
327   __ shrptr(tmp1, CardTable::card_shift());                       // tmp1 := card address relative to card table base
328 
329   Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()));
330   __ addptr(tmp1, card_table_addr);                               // tmp1 := card address
331   if (UseCondCardMark) {
332     __ cmpb(Address(tmp1, 0), G1CardTable::clean_card_val());     // *(card address) == clean_card_val?
333     __ jcc(Assembler::notEqual, done);
334   }
335   // Storing a region crossing, non-null oop, card is clean.
336   // Dirty card.
337   __ movb(Address(tmp1, 0), G1CardTable::dirty_card_val());       // *(card address) := dirty_card_val
338 }
339 
340 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
341                                                   Register store_addr,
342                                                   Register new_val,
343                                                   Register tmp) {
344   Label done;
345   generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, done, true /* new_val_may_be_null */);
346   __ bind(done);
347 }
348 
349 #if defined(COMPILER2)
350 
351 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
352   SaveLiveRegisters save_registers(masm, stub);
353   if (c_rarg0 != arg) {
354     __ mov(c_rarg0, arg);
355   }
356   __ mov(c_rarg1, r15_thread);
357   // rax is a caller-saved, non-argument-passing register, so it does not
358   // interfere with c_rarg0 or c_rarg1. If it contained any live value before
359   // entering this stub, it is saved at this point, and restored after the
360   // call. If it did not contain any live value, it is free to be used. In
361   // either case, it is safe to use it here as a call scratch register.
362   __ call(RuntimeAddress(runtime_path), rax);
363 }
364 
365 void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
366                                                     Register obj,
367                                                     Register pre_val,
368                                                     Register tmp,
369                                                     G1PreBarrierStubC2* stub) {
370   const Register thread = r15_thread;
371 
372   assert(pre_val != noreg, "check this code");
373   if (obj != noreg) {
374     assert_different_registers(obj, pre_val, tmp);
375   }
376 
377   stub->initialize_registers(obj, pre_val, thread, tmp);
378 
379   generate_pre_barrier_fast_path(masm, thread);
380   // If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
381   __ jcc(Assembler::notEqual, *stub->entry());
382 
383   __ bind(*stub->continuation());
384 }
385 
386 void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
387                                                          G1PreBarrierStubC2* stub) const {
388   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
389   Label runtime;
390   Register obj = stub->obj();
391   Register pre_val = stub->pre_val();
392   Register thread = stub->thread();
393   Register tmp = stub->tmp1();
394   assert(stub->tmp2() == noreg, "not needed in this platform");
395 
396   __ bind(*stub->entry());
397   generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, *stub->continuation(), runtime);
398 
399   __ bind(runtime);
400   generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
401   __ jmp(*stub->continuation());
402 }
403 
404 void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
405                                                      Register store_addr,
406                                                      Register new_val,
407                                                      Register tmp,
408                                                      bool new_val_may_be_null) {
409   Label done;
410   generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, done, new_val_may_be_null);
411   __ bind(done);
412 }
413 
414 #endif // COMPILER2
415 
416 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
417                                          Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
418   bool in_heap = (decorators & IN_HEAP) != 0;
419   bool as_normal = (decorators & AS_NORMAL) != 0;
420 
421   bool needs_pre_barrier = as_normal;
422   bool needs_post_barrier = val != noreg && in_heap;
423 
424   // flatten object address if needed
425   // We do it regardless of precise because we need the registers
426   if (dst.index() == noreg && dst.disp() == 0) {
427     if (dst.base() != tmp1) {
428       __ movptr(tmp1, dst.base());
429     }
430   } else {
431     __ lea(tmp1, dst);
432   }
433 
434   if (needs_pre_barrier) {
435     g1_write_barrier_pre(masm /*masm*/,
436                          tmp1 /* obj */,
437                          tmp2 /* pre_val */,
438                          tmp3  /* tmp */,
439                          val != noreg /* tosca_live */,
440                          false /* expand_call */);
441   }
442   if (val == noreg) {
443     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
444   } else {
445     Register new_val = val;
446     if (needs_post_barrier) {
447       // G1 barrier needs uncompressed oop for region cross check.
448       if (UseCompressedOops) {
449         new_val = tmp2;
450         __ movptr(new_val, val);
451       }
452     }
453     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
454     if (needs_post_barrier) {
455       g1_write_barrier_post(masm /*masm*/,
456                             tmp1 /* store_adr */,
457                             new_val /* new_val */,
458                             tmp3 /* tmp */);
459     }
460   }
461 }
462 
463 #ifdef COMPILER1
464 
465 #undef __
466 #define __ ce->masm()->
467 
468 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
469   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
470   // At this point we know that marking is in progress.
471   // If do_load() is true then we have to emit the
472   // load of the previous value; otherwise it has already
473   // been loaded into _pre_val.
474 
475   __ bind(*stub->entry());
476   assert(stub->pre_val()->is_register(), "Precondition.");
477 
478   Register pre_val_reg = stub->pre_val()->as_register();
479 
480   if (stub->do_load()) {
481     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
482   }
483 
484   __ cmpptr(pre_val_reg, NULL_WORD);
485   __ jcc(Assembler::equal, *stub->continuation());
486   ce->store_parameter(stub->pre_val()->as_register(), 0);
487   __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
488   __ jmp(*stub->continuation());
489 
490 }
491 
492 #undef __
493 
494 void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
495                                                      Register store_addr,
496                                                      Register new_val,
497                                                      Register thread,
498                                                      Register tmp1,
499                                                      Register tmp2 /* unused on x86 */) {
500   Label done;
501   generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, done, true /* new_val_may_be_null */);
502   masm->bind(done);
503 }
504 
505 #define __ sasm->
506 
507 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
508   // Generated code assumes that buffer index is pointer sized.
509   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
510 
511   __ prologue("g1_pre_barrier", false);
512   // arg0 : previous value of memory
513 
514   __ push_ppx(rax);
515   __ push_ppx(rdx);
516 
517   const Register pre_val = rax;
518   const Register thread = r15_thread;
519   const Register tmp = rdx;
520 
521   Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
522   Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
523   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
524 
525   Label done;
526   Label runtime;
527 
528   // Is marking still active?
529   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
530     __ cmpl(queue_active, 0);
531   } else {
532     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
533     __ cmpb(queue_active, 0);
534   }
535   __ jcc(Assembler::equal, done);
536 
537   // Can we store original value in the thread's buffer?
538 
539   __ movptr(tmp, queue_index);
540   __ testptr(tmp, tmp);
541   __ jcc(Assembler::zero, runtime);
542   __ subptr(tmp, wordSize);
543   __ movptr(queue_index, tmp);
544   __ addptr(tmp, buffer);
545 
546   // prev_val (rax)
547   __ load_parameter(0, pre_val);
548   __ movptr(Address(tmp, 0), pre_val);
549   __ jmp(done);
550 
551   __ bind(runtime);
552 
553   __ push_call_clobbered_registers();
554 
555   // load the pre-value
556   __ load_parameter(0, rcx);
557   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread);
558 
559   __ pop_call_clobbered_registers();
560 
561   __ bind(done);
562 
563   __ pop_ppx(rdx);
564   __ pop_ppx(rax);
565 
566   __ epilogue();
567 }
568 
569 #undef __
570 
571 #endif // COMPILER1