1 /*
  2  * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "asm/macroAssembler.inline.hpp"
 27 #if INCLUDE_CDS
 28 #include "code/SCCache.hpp"
 29 #endif
 30 #include "gc/g1/g1BarrierSet.hpp"
 31 #include "gc/g1/g1BarrierSetAssembler.hpp"
 32 #include "gc/g1/g1BarrierSetRuntime.hpp"
 33 #include "gc/g1/g1CardTable.hpp"
 34 #include "gc/g1/g1HeapRegion.hpp"
 35 #include "gc/g1/g1ThreadLocalData.hpp"
 36 #include "gc/shared/collectedHeap.hpp"
 37 #include "interpreter/interp_masm.hpp"
 38 #include "runtime/javaThread.hpp"
 39 #include "runtime/sharedRuntime.hpp"
 40 #ifdef COMPILER1
 41 #include "c1/c1_LIRAssembler.hpp"
 42 #include "c1/c1_MacroAssembler.hpp"
 43 #include "gc/g1/c1/g1BarrierSetC1.hpp"
 44 #endif
 45 
 46 #define __ masm->
 47 
 48 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
 49                                                             Register addr, Register count, RegSet saved_regs) {
 50   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 51   if (!dest_uninitialized) {
 52     Label done;
 53     Address in_progress(rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
 54 
 55     // Is marking active?
 56     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 57       __ ldrw(rscratch1, in_progress);
 58     } else {
 59       assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 60       __ ldrb(rscratch1, in_progress);
 61     }
 62     __ cbzw(rscratch1, done);
 63 
 64     __ push(saved_regs, sp);
 65     if (count == c_rarg0) {
 66       if (addr == c_rarg1) {
 67         // exactly backwards!!
 68         __ mov(rscratch1, c_rarg0);
 69         __ mov(c_rarg0, c_rarg1);
 70         __ mov(c_rarg1, rscratch1);
 71       } else {
 72         __ mov(c_rarg1, count);
 73         __ mov(c_rarg0, addr);
 74       }
 75     } else {
 76       __ mov(c_rarg0, addr);
 77       __ mov(c_rarg1, count);
 78     }
 79     if (UseCompressedOops) {
 80       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
 81     } else {
 82       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
 83     }
 84     __ pop(saved_regs, sp);
 85 
 86     __ bind(done);
 87   }
 88 }
 89 
 90 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 91                                                              Register start, Register count, Register scratch, RegSet saved_regs) {
 92   __ push(saved_regs, sp);
 93   assert_different_registers(start, count, scratch);
 94   assert_different_registers(c_rarg0, count);
 95   __ mov(c_rarg0, start);
 96   __ mov(c_rarg1, count);
 97   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
 98   __ pop(saved_regs, sp);
 99 }
100 
101 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
102                                                  Register obj,
103                                                  Register pre_val,
104                                                  Register thread,
105                                                  Register tmp1,
106                                                  Register tmp2,
107                                                  bool tosca_live,
108                                                  bool expand_call) {
109   // If expand_call is true then we expand the call_VM_leaf macro
110   // directly to skip generating the check by
111   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
112 
113   assert(thread == rthread, "must be");
114 
115   Label done;
116   Label runtime;
117 
118   assert_different_registers(obj, pre_val, tmp1, tmp2);
119   assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
120 
121   Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
122   Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
123   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
124 
125   // Is marking active?
126   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
127     __ ldrw(tmp1, in_progress);
128   } else {
129     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
130     __ ldrb(tmp1, in_progress);
131   }
132   __ cbzw(tmp1, done);
133 
134   // Do we need to load the previous value?
135   if (obj != noreg) {
136     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
137   }
138 
139   // Is the previous value null?
140   __ cbz(pre_val, done);
141 
142   // Can we store original value in the thread's buffer?
143   // Is index == 0?
144   // (The index field is typed as size_t.)
145 
146   __ ldr(tmp1, index);                      // tmp := *index_adr
147   __ cbz(tmp1, runtime);                    // tmp == 0?
148                                         // If yes, goto runtime
149 
150   __ sub(tmp1, tmp1, wordSize);             // tmp := tmp - wordSize
151   __ str(tmp1, index);                      // *index_adr := tmp
152   __ ldr(tmp2, buffer);
153   __ add(tmp1, tmp1, tmp2);                 // tmp := tmp + *buffer_adr
154 
155   // Record the previous value
156   __ str(pre_val, Address(tmp1, 0));
157   __ b(done);
158 
159   __ bind(runtime);
160 
161   __ push_call_clobbered_registers();
162 
163   // Calling the runtime using the regular call_VM_leaf mechanism generates
164   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
165   // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
166   //
167   // If we care generating the pre-barrier without a frame (e.g. in the
168   // intrinsified Reference.get() routine) then rfp might be pointing to
169   // the caller frame and so this check will most likely fail at runtime.
170   //
171   // Expanding the call directly bypasses the generation of the check.
172   // So when we do not have have a full interpreter frame on the stack
173   // expand_call should be passed true.
174 
175   if (expand_call) {
176     assert(pre_val != c_rarg1, "smashed arg");
177     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
178   } else {
179     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
180   }
181 
182   __ pop_call_clobbered_registers();
183 
184   __ bind(done);
185 
186 }
187 
188 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
189                                                   Register store_addr,
190                                                   Register new_val,
191                                                   Register thread,
192                                                   Register tmp1,
193                                                   Register tmp2) {
194   assert(thread == rthread, "must be");
195   assert_different_registers(store_addr, new_val, thread, tmp1, tmp2,
196                              rscratch1);
197   assert(store_addr != noreg && new_val != noreg && tmp1 != noreg
198          && tmp2 != noreg, "expecting a register");
199 
200   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
201   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
202 
203   BarrierSet* bs = BarrierSet::barrier_set();
204   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
205   CardTable* ct = ctbs->card_table();
206 
207   Label done;
208   Label runtime;
209 
210   // Does store cross heap regions?
211 
212 #if INCLUDE_CDS
213   // AOT code needs to load the barrier grain shift from the aot
214   // runtime constants area in the code cache otherwise we can compile
215   // it as an immediate operand
216   if (SCCache::is_on_for_write()) {
217     address grain_shift_address = (address)AOTRuntimeConstants::grain_shift_address();
218     __ eor(tmp1, store_addr, new_val);
219     __ lea(tmp2, ExternalAddress(grain_shift_address));
220     __ ldrb(tmp2, tmp2);
221     __ lsrv(tmp1, tmp1, tmp2);
222     __ cbz(tmp1, done);
223   } else
224 #endif
225   {
226     __ eor(tmp1, store_addr, new_val);
227     __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);
228     __ cbz(tmp1, done);
229   }
230   // crosses regions, storing null?
231 
232   __ cbz(new_val, done);
233 
234   // storing region crossing non-null, is card already dirty?
235 
236   const Register card_addr = tmp1;
237 
238 #if INCLUDE_CDS
239   // AOT code needs to load the barrier card shift from the aot
240   // runtime constants area in the code cache otherwise we can compile
241   // it as an immediate operand
242   if (SCCache::is_on_for_write()) {
243     address card_shift_address = (address)AOTRuntimeConstants::card_shift_address();
244     __ lea(tmp2, ExternalAddress(card_shift_address));
245     __ ldrb(tmp2, tmp2);
246     __ lsrv(card_addr, store_addr, tmp2);
247   } else
248 #endif
249   {
250     __ lsr(card_addr, store_addr, CardTable::card_shift());
251   }
252 
253   // get the address of the card
254   __ load_byte_map_base(tmp2);
255   __ add(card_addr, card_addr, tmp2);
256   __ ldrb(tmp2, Address(card_addr));
257   __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val());
258   __ br(Assembler::EQ, done);
259 
260   assert((int)CardTable::dirty_card_val() == 0, "must be 0");
261 
262   __ membar(Assembler::StoreLoad);
263 
264   __ ldrb(tmp2, Address(card_addr));
265   __ cbzw(tmp2, done);
266 
267   // storing a region crossing, non-null oop, card is clean.
268   // dirty card and log.
269 
270   __ strb(zr, Address(card_addr));
271 
272   __ ldr(rscratch1, queue_index);
273   __ cbz(rscratch1, runtime);
274   __ sub(rscratch1, rscratch1, wordSize);
275   __ str(rscratch1, queue_index);
276 
277   __ ldr(tmp2, buffer);
278   __ str(card_addr, Address(tmp2, rscratch1));
279   __ b(done);
280 
281   __ bind(runtime);
282   // save the live input values
283   RegSet saved = RegSet::of(store_addr);
284   __ push(saved, sp);
285   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
286   __ pop(saved, sp);
287 
288   __ bind(done);
289 }
290 
291 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
292                                     Register dst, Address src, Register tmp1, Register tmp2) {
293   bool on_oop = is_reference_type(type);
294   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
295   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
296   bool on_reference = on_weak || on_phantom;
297   ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
298   if (on_oop && on_reference) {
299     // LR is live.  It must be saved around calls.
300     __ enter(/*strip_ret_addr*/true); // barrier may call runtime
301     // Generate the G1 pre-barrier code to log the value of
302     // the referent field in an SATB buffer.
303     g1_write_barrier_pre(masm /* masm */,
304                          noreg /* obj */,
305                          dst /* pre_val */,
306                          rthread /* thread */,
307                          tmp1 /* tmp1 */,
308                          tmp2 /* tmp2 */,
309                          true /* tosca_live */,
310                          true /* expand_call */);
311     __ leave();
312   }
313 }
314 
315 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
316                                          Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
317   // flatten object address if needed
318   if (dst.index() == noreg && dst.offset() == 0) {
319     if (dst.base() != tmp3) {
320       __ mov(tmp3, dst.base());
321     }
322   } else {
323     __ lea(tmp3, dst);
324   }
325 
326   g1_write_barrier_pre(masm,
327                        tmp3 /* obj */,
328                        tmp2 /* pre_val */,
329                        rthread /* thread */,
330                        tmp1  /* tmp1 */,
331                        rscratch2  /* tmp2 */,
332                        val != noreg /* tosca_live */,
333                        false /* expand_call */);
334 
335   if (val == noreg) {
336     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
337   } else {
338     // G1 barrier needs uncompressed oop for region cross check.
339     Register new_val = val;
340     if (UseCompressedOops) {
341       new_val = rscratch2;
342       __ mov(new_val, val);
343     }
344     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
345     g1_write_barrier_post(masm,
346                           tmp3 /* store_adr */,
347                           new_val /* new_val */,
348                           rthread /* thread */,
349                           tmp1 /* tmp1 */,
350                           tmp2 /* tmp2 */);
351   }
352 
353 }
354 
355 #ifdef COMPILER1
356 
357 #undef __
358 #define __ ce->masm()->
359 
360 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
361   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
362   // At this point we know that marking is in progress.
363   // If do_load() is true then we have to emit the
364   // load of the previous value; otherwise it has already
365   // been loaded into _pre_val.
366 
367   __ bind(*stub->entry());
368 
369   assert(stub->pre_val()->is_register(), "Precondition.");
370 
371   Register pre_val_reg = stub->pre_val()->as_register();
372 
373   if (stub->do_load()) {
374     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
375   }
376   __ cbz(pre_val_reg, *stub->continuation());
377   ce->store_parameter(stub->pre_val()->as_register(), 0);
378   __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
379   __ b(*stub->continuation());
380 }
381 
382 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
383   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
384   __ bind(*stub->entry());
385   assert(stub->addr()->is_register(), "Precondition.");
386   assert(stub->new_val()->is_register(), "Precondition.");
387   Register new_val_reg = stub->new_val()->as_register();
388   __ cbz(new_val_reg, *stub->continuation());
389   ce->store_parameter(stub->addr()->as_pointer_register(), 0);
390   __ far_call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
391   __ b(*stub->continuation());
392 }
393 
394 #undef __
395 
396 #define __ sasm->
397 
398 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
399   __ prologue("g1_pre_barrier", false);
400 
401   // arg0 : previous value of memory
402 
403   BarrierSet* bs = BarrierSet::barrier_set();
404 
405   const Register pre_val = r0;
406   const Register thread = rthread;
407   const Register tmp = rscratch1;
408 
409   Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
410   Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
411   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
412 
413   Label done;
414   Label runtime;
415 
416   // Is marking still active?
417   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
418     __ ldrw(tmp, in_progress);
419   } else {
420     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
421     __ ldrb(tmp, in_progress);
422   }
423   __ cbzw(tmp, done);
424 
425   // Can we store original value in the thread's buffer?
426   __ ldr(tmp, queue_index);
427   __ cbz(tmp, runtime);
428 
429   __ sub(tmp, tmp, wordSize);
430   __ str(tmp, queue_index);
431   __ ldr(rscratch2, buffer);
432   __ add(tmp, tmp, rscratch2);
433   __ load_parameter(0, rscratch2);
434   __ str(rscratch2, Address(tmp, 0));
435   __ b(done);
436 
437   __ bind(runtime);
438   __ push_call_clobbered_registers();
439   __ load_parameter(0, pre_val);
440   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
441   __ pop_call_clobbered_registers();
442   __ bind(done);
443 
444   __ epilogue();
445 }
446 
447 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
448   __ prologue("g1_post_barrier", false);
449 
450   // arg0: store_address
451   Address store_addr(rfp, 2*BytesPerWord);
452 
453   BarrierSet* bs = BarrierSet::barrier_set();
454   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
455   CardTable* ct = ctbs->card_table();
456 
457   Label done;
458   Label runtime;
459 
460   // At this point we know new_value is non-null and the new_value crosses regions.
461   // Must check to see if card is already dirty
462 
463   const Register thread = rthread;
464 
465   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
466   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
467 
468   const Register card_offset = rscratch2;
469   // LR is free here, so we can use it to hold the byte_map_base.
470   const Register byte_map_base = lr;
471 
472   assert_different_registers(card_offset, byte_map_base, rscratch1);
473 
474   __ load_parameter(0, card_offset);
475   __ lsr(card_offset, card_offset, CardTable::card_shift());
476   __ load_byte_map_base(byte_map_base);
477   __ ldrb(rscratch1, Address(byte_map_base, card_offset));
478   __ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val());
479   __ br(Assembler::EQ, done);
480 
481   assert((int)CardTable::dirty_card_val() == 0, "must be 0");
482 
483   __ membar(Assembler::StoreLoad);
484   __ ldrb(rscratch1, Address(byte_map_base, card_offset));
485   __ cbzw(rscratch1, done);
486 
487   // storing region crossing non-null, card is clean.
488   // dirty card and log.
489   __ strb(zr, Address(byte_map_base, card_offset));
490 
491   // Convert card offset into an address in card_addr
492   Register card_addr = card_offset;
493   __ add(card_addr, byte_map_base, card_addr);
494 
495   __ ldr(rscratch1, queue_index);
496   __ cbz(rscratch1, runtime);
497   __ sub(rscratch1, rscratch1, wordSize);
498   __ str(rscratch1, queue_index);
499 
500   // Reuse LR to hold buffer_addr
501   const Register buffer_addr = lr;
502 
503   __ ldr(buffer_addr, buffer);
504   __ str(card_addr, Address(buffer_addr, rscratch1));
505   __ b(done);
506 
507   __ bind(runtime);
508   __ push_call_clobbered_registers();
509   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
510   __ pop_call_clobbered_registers();
511   __ bind(done);
512   __ epilogue();
513 }
514 
515 #undef __
516 
517 #endif // COMPILER1