1 /*
  2  * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  *
 24  */
 25 
 26 #include "precompiled.hpp"
 27 #include "asm/macroAssembler.inline.hpp"
 28 #include "gc/g1/g1BarrierSet.hpp"
 29 #include "gc/g1/g1BarrierSetAssembler.hpp"
 30 #include "gc/g1/g1BarrierSetRuntime.hpp"
 31 #include "gc/g1/g1CardTable.hpp"
 32 #include "gc/g1/g1ThreadLocalData.hpp"
 33 #include "gc/g1/heapRegion.hpp"
 34 #include "gc/shared/collectedHeap.hpp"
 35 #include "interpreter/interp_masm.hpp"
 36 #include "runtime/sharedRuntime.hpp"
 37 #include "runtime/thread.hpp"
 38 #ifdef COMPILER1
 39 #include "c1/c1_LIRAssembler.hpp"
 40 #include "c1/c1_MacroAssembler.hpp"
 41 #include "gc/g1/c1/g1BarrierSetC1.hpp"
 42 #endif
 43 
 44 #define __ masm->
 45 
 46 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
 47                                                             Register addr, Register count, RegSet saved_regs) {
 48   assert_cond(masm != NULL);
 49   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 50   if (!dest_uninitialized) {
 51     Label done;
 52     Address in_progress(xthread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
 53 
 54     // Is marking active?
 55     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 56       __ lwu(t0, in_progress);
 57     } else {
 58       assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 59       __ lbu(t0, in_progress);
 60     }
 61     __ beqz(t0, done);
 62 
 63     __ push_reg(saved_regs, sp);
 64     if (count == c_rarg0) {
 65       if (addr == c_rarg1) {
 66         // exactly backwards!!
 67         __ mv(t0, c_rarg0);
 68         __ mv(c_rarg0, c_rarg1);
 69         __ mv(c_rarg1, t0);
 70       } else {
 71         __ mv(c_rarg1, count);
 72         __ mv(c_rarg0, addr);
 73       }
 74     } else {
 75       __ mv(c_rarg0, addr);
 76       __ mv(c_rarg1, count);
 77     }
 78     if (UseCompressedOops) {
 79       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
 80     } else {
 81       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
 82     }
 83     __ pop_reg(saved_regs, sp);
 84 
 85     __ bind(done);
 86   }
 87 }
 88 
 89 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 90                                                              Register start, Register count, Register tmp, RegSet saved_regs) {
 91   assert_cond(masm != NULL);
 92   __ push_reg(saved_regs, sp);
 93   assert_different_registers(start, count, tmp);
 94   assert_different_registers(c_rarg0, count);
 95   __ mv(c_rarg0, start);
 96   __ mv(c_rarg1, count);
 97   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
 98   __ pop_reg(saved_regs, sp);
 99 }
100 
101 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
102                                                  Register obj,
103                                                  Register pre_val,
104                                                  Register thread,
105                                                  Register tmp,
106                                                  bool tosca_live,
107                                                  bool expand_call) {
108   // If expand_call is true then we expand the call_VM_leaf macro
109   // directly to skip generating the check by
110   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
111 
112   assert_cond(masm != NULL);
113   assert(thread == xthread, "must be");
114 
115   Label done;
116   Label runtime;
117 
118   assert_different_registers(obj, pre_val, tmp, t0);
119   assert(pre_val != noreg &&  tmp != noreg, "expecting a register");
120 
121   Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
122   Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
123   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
124 
125   // Is marking active?
126   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { // 4-byte width
127     __ lwu(tmp, in_progress);
128   } else {
129     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
130     __ lbu(tmp, in_progress);
131   }
132   __ beqz(tmp, done);
133 
134   // Do we need to load the previous value?
135   if (obj != noreg) {
136     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
137   }
138 
139   // Is the previous value null?
140   __ beqz(pre_val, done);
141 
142   // Can we store original value in the thread's buffer?
143   // Is index == 0?
144   // (The index field is typed as size_t.)
145 
146   __ ld(tmp, index);                       // tmp := *index_adr
147   __ beqz(tmp, runtime);                   // tmp == 0?
148                                            // If yes, goto runtime
149 
150   __ sub(tmp, tmp, wordSize);              // tmp := tmp - wordSize
151   __ sd(tmp, index);                       // *index_adr := tmp
152   __ ld(t0, buffer);
153   __ add(tmp, tmp, t0);                    // tmp := tmp + *buffer_adr
154 
155   // Record the previous value
156   __ sd(pre_val, Address(tmp, 0));
157   __ j(done);
158 
159   __ bind(runtime);
160   // save the live input values
161   RegSet saved = RegSet::of(pre_val);
162   if (tosca_live) { saved += RegSet::of(x10); }
163   if (obj != noreg) { saved += RegSet::of(obj); }
164 
165   __ push_reg(saved, sp);
166 
167   if (expand_call) {
168     assert(pre_val != c_rarg1, "smashed arg");
169     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
170   } else {
171     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
172   }
173 
174   __ pop_reg(saved, sp);
175 
176   __ bind(done);
177 
178 }
179 
180 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
181                                                   Register store_addr,
182                                                   Register new_val,
183                                                   Register thread,
184                                                   Register tmp,
185                                                   Register tmp2) {
186   assert_cond(masm != NULL);
187   assert(thread == xthread, "must be");
188   assert_different_registers(store_addr, new_val, thread, tmp, tmp2,
189                              t0);
190   assert(store_addr != noreg && new_val != noreg && tmp != noreg &&
191          tmp2 != noreg, "expecting a register");
192 
193   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
194   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
195 
196   BarrierSet* bs = BarrierSet::barrier_set();
197   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
198   CardTable* ct = ctbs->card_table();
199 
200   Label done;
201   Label runtime;
202 
203   // Does store cross heap regions?
204 
205   __ xorr(tmp, store_addr, new_val);
206   __ srli(tmp, tmp, HeapRegion::LogOfHRGrainBytes);
207   __ beqz(tmp, done);
208 
209   // crosses regions, storing NULL?
210 
211   __ beqz(new_val, done);
212 
213   // storing region crossing non-NULL, is card already dirty?
214 
215   ExternalAddress cardtable((address) ct->byte_map_base());
216   const Register card_addr = tmp;
217 
218   __ srli(card_addr, store_addr, CardTable::card_shift);
219 
220   // get the address of the card
221   __ load_byte_map_base(tmp2);
222   __ add(card_addr, card_addr, tmp2);
223   __ lbu(tmp2, Address(card_addr));
224   __ mv(t0, (int)G1CardTable::g1_young_card_val());
225   __ beq(tmp2, t0, done);
226 
227   assert((int)CardTable::dirty_card_val() == 0, "must be 0");
228 
229   __ membar(MacroAssembler::StoreLoad);
230 
231   __ lbu(tmp2, Address(card_addr));
232   __ beqz(tmp2, done);
233 
234   // storing a region crossing, non-NULL oop, card is clean.
235   // dirty card and log.
236 
237   __ sb(zr, Address(card_addr));
238 
239   __ ld(t0, queue_index);
240   __ beqz(t0, runtime);
241   __ sub(t0, t0, wordSize);
242   __ sd(t0, queue_index);
243 
244   __ ld(tmp2, buffer);
245   __ add(t0, tmp2, t0);
246   __ sd(card_addr, Address(t0, 0));
247   __ j(done);
248 
249   __ bind(runtime);
250   // save the live input values
251   RegSet saved = RegSet::of(store_addr);
252   __ push_reg(saved, sp);
253   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
254   __ pop_reg(saved, sp);
255 
256   __ bind(done);
257 }
258 
259 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
260                                     Register dst, Address src, Register tmp1, Register tmp_thread) {
261   assert_cond(masm != NULL);
262   bool on_oop = is_reference_type(type);
263   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
264   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
265   bool on_reference = on_weak || on_phantom;
266   ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
267   if (on_oop && on_reference) {
268     // LR is live.  It must be saved around calls.
269     __ enter(); // barrier may call runtime
270     // Generate the G1 pre-barrier code to log the value of
271     // the referent field in an SATB buffer.
272     g1_write_barrier_pre(masm /* masm */,
273                          noreg /* obj */,
274                          dst /* pre_val */,
275                          xthread /* thread */,
276                          tmp1 /* tmp */,
277                          true /* tosca_live */,
278                          true /* expand_call */);
279     __ leave();
280   }
281 }
282 
283 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
284                                          Address dst, Register val, Register tmp1, Register tmp2) {
285   assert_cond(masm != NULL);
286   // flatten object address if needed
287   if (dst.offset() == 0) {
288     if (dst.base() != x13) {
289       __ mv(x13, dst.base());
290     }
291   } else {
292     __ la(x13, dst);
293   }
294 
295   g1_write_barrier_pre(masm,
296                        x13 /* obj */,
297                        tmp2 /* pre_val */,
298                        xthread /* thread */,
299                        tmp1  /* tmp */,
300                        val != noreg /* tosca_live */,
301                        false /* expand_call */);
302 
303   if (val == noreg) {
304     BarrierSetAssembler::store_at(masm, decorators, type, Address(x13, 0), noreg, noreg, noreg);
305   } else {
306     // G1 barrier needs uncompressed oop for region cross check.
307     Register new_val = val;
308     if (UseCompressedOops) {
309       new_val = t1;
310       __ mv(new_val, val);
311     }
312     BarrierSetAssembler::store_at(masm, decorators, type, Address(x13, 0), val, noreg, noreg);
313     g1_write_barrier_post(masm,
314                           x13 /* store_adr */,
315                           new_val /* new_val */,
316                           xthread /* thread */,
317                           tmp1 /* tmp */,
318                           tmp2 /* tmp2 */);
319   }
320 }
321 
322 #ifdef COMPILER1
323 
324 #undef __
325 #define __ ce->masm()->
326 
327 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
328   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
329 
330   // At this point we know that marking is in progress.
331   // If do_load() is true then we have to emit the
332   // load of the previous value; otherwise it has already
333   // been loaded into _pre_val.
334   __ bind(*stub->entry());
335 
336   assert(stub->pre_val()->is_register(), "Precondition.");
337 
338   Register pre_val_reg = stub->pre_val()->as_register();
339 
340   if (stub->do_load()) {
341     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /* wide */);
342   }
343   __ beqz(pre_val_reg, *stub->continuation(), /* is_far */ true);
344   ce->store_parameter(stub->pre_val()->as_register(), 0);
345   __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
346   __ j(*stub->continuation());
347 }
348 
349 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
350   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
351   __ bind(*stub->entry());
352   assert(stub->addr()->is_register(), "Precondition");
353   assert(stub->new_val()->is_register(), "Precondition");
354   Register new_val_reg = stub->new_val()->as_register();
355   __ beqz(new_val_reg, *stub->continuation(), /* is_far */ true);
356   ce->store_parameter(stub->addr()->as_pointer_register(), 0);
357   __ far_call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
358   __ j(*stub->continuation());
359 }
360 
361 #undef __
362 
363 #define __ sasm->
364 
365 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
366   __ prologue("g1_pre_barrier", false);
367 
368   BarrierSet* bs = BarrierSet::barrier_set();
369 
370   // arg0 : previous value of memory
371   const Register pre_val = x10;
372   const Register thread = xthread;
373   const Register tmp = t0;
374 
375   Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
376   Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
377   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
378 
379   Label done;
380   Label runtime;
381 
382   // Is marking still active?
383   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {  // 4-byte width
384     __ lwu(tmp, in_progress);
385   } else {
386     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
387     __ lbu(tmp, in_progress);
388   }
389   __ beqz(tmp, done);
390 
391   // Can we store original value in the thread's buffer?
392   __ ld(tmp, queue_index);
393   __ beqz(tmp, runtime);
394 
395   __ sub(tmp, tmp, wordSize);
396   __ sd(tmp, queue_index);
397   __ ld(t1, buffer);
398   __ add(tmp, tmp, t1);
399   __ load_parameter(0, t1);
400   __ sd(t1, Address(tmp, 0));
401   __ j(done);
402 
403   __ bind(runtime);
404   __ push_call_clobbered_registers();
405   __ load_parameter(0, pre_val);
406   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
407   __ pop_call_clobbered_registers();
408   __ bind(done);
409 
410   __ epilogue();
411 }
412 
413 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
414   __ prologue("g1_post_barrier", false);
415 
416   // arg0 : store_address
417   Address store_addr(fp, 2 * BytesPerWord); // 2 BytesPerWord from fp
418 
419   BarrierSet* bs = BarrierSet::barrier_set();
420   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
421   CardTable* ct = ctbs->card_table();
422 
423   Label done;
424   Label runtime;
425 
426   // At this point we know new_value is non-NULL and the new_value crosses regions.
427   // Must check to see if card is already dirty
428   const Register thread = xthread;
429 
430   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
431   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
432 
433   const Register card_offset = t1;
434   // LR is free here, so we can use it to hold the byte_map_base.
435   const Register byte_map_base = lr;
436 
437   assert_different_registers(card_offset, byte_map_base, t0);
438 
439   __ load_parameter(0, card_offset);
440   __ srli(card_offset, card_offset, CardTable::card_shift);
441   __ load_byte_map_base(byte_map_base);
442 
443   // Convert card offset into an address in card_addr
444   Register card_addr = card_offset;
445   __ add(card_addr, byte_map_base, card_addr);
446 
447   __ lbu(t0, Address(card_addr, 0));
448   __ sub(t0, t0, (int)G1CardTable::g1_young_card_val());
449   __ beqz(t0, done);
450 
451   assert((int)CardTable::dirty_card_val() == 0, "must be 0");
452 
453   __ membar(MacroAssembler::StoreLoad);
454   __ lbu(t0, Address(card_addr, 0));
455   __ beqz(t0, done);
456 
457   // storing region crossing non-NULL, card is clean.
458   // dirty card and log.
459   __ sb(zr, Address(card_addr, 0));
460 
461   __ ld(t0, queue_index);
462   __ beqz(t0, runtime);
463   __ sub(t0, t0, wordSize);
464   __ sd(t0, queue_index);
465 
466   // Reuse LR to hold buffer_addr
467   const Register buffer_addr = lr;
468 
469   __ ld(buffer_addr, buffer);
470   __ add(t0, buffer_addr, t0);
471   __ sd(card_addr, Address(t0, 0));
472   __ j(done);
473 
474   __ bind(runtime);
475   __ push_call_clobbered_registers();
476   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
477   __ pop_call_clobbered_registers();
478   __ bind(done);
479   __ epilogue();
480 }
481 
482 #undef __
483 
484 #endif // COMPILER1