1 /*
  2  * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "asm/macroAssembler.inline.hpp"
 27 #include "gc/g1/g1BarrierSet.hpp"
 28 #include "gc/g1/g1BarrierSetAssembler.hpp"
 29 #include "gc/g1/g1BarrierSetRuntime.hpp"
 30 #include "gc/g1/g1CardTable.hpp"
 31 #include "gc/g1/g1HeapRegion.hpp"
 32 #include "gc/g1/g1ThreadLocalData.hpp"
 33 #include "interpreter/interp_masm.hpp"
 34 #include "runtime/sharedRuntime.hpp"
 35 #include "utilities/debug.hpp"
 36 #include "utilities/macros.hpp"
 37 #ifdef COMPILER1
 38 #include "c1/c1_LIRAssembler.hpp"
 39 #include "c1/c1_MacroAssembler.hpp"
 40 #include "gc/g1/c1/g1BarrierSetC1.hpp"
 41 #endif // COMPILER1
 42 #ifdef COMPILER2
 43 #include "gc/g1/c2/g1BarrierSetC2.hpp"
 44 #endif // COMPILER2
 45 
 46 #define __ masm->
 47 
 48 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
 49                                                             Register addr, Register count) {
 50   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 51 
 52   if (!dest_uninitialized) {
 53     Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
 54 #ifndef _LP64
 55     __ push(thread);
 56     __ get_thread(thread);
 57 #endif
 58 
 59     Label filtered;
 60     Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
 61     // Is marking active?
 62     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 63       __ cmpl(in_progress, 0);
 64     } else {
 65       assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 66       __ cmpb(in_progress, 0);
 67     }
 68 
 69     NOT_LP64(__ pop(thread);)
 70 
 71     __ jcc(Assembler::equal, filtered);
 72 
 73     __ push_call_clobbered_registers(false /* save_fpu */);
 74 #ifdef _LP64
 75     if (count == c_rarg0) {
 76       if (addr == c_rarg1) {
 77         // exactly backwards!!
 78         __ xchgptr(c_rarg1, c_rarg0);
 79       } else {
 80         __ movptr(c_rarg1, count);
 81         __ movptr(c_rarg0, addr);
 82       }
 83     } else {
 84       __ movptr(c_rarg0, addr);
 85       __ movptr(c_rarg1, count);
 86     }
 87     if (UseCompressedOops) {
 88       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
 89     } else {
 90       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
 91     }
 92 #else
 93     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry),
 94                     addr, count);
 95 #endif
 96     __ pop_call_clobbered_registers(false /* save_fpu */);
 97 
 98     __ bind(filtered);
 99   }
100 }
101 
102 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
103                                                              Register addr, Register count, Register tmp) {
104   __ push_call_clobbered_registers(false /* save_fpu */);
105 #ifdef _LP64
106   if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
107     assert_different_registers(c_rarg1, addr);
108     __ mov(c_rarg1, count);
109     __ mov(c_rarg0, addr);
110   } else {
111     assert_different_registers(c_rarg0, count);
112     __ mov(c_rarg0, addr);
113     __ mov(c_rarg1, count);
114   }
115   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
116 #else
117   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry),
118                   addr, count);
119 #endif
120   __ pop_call_clobbered_registers(false /* save_fpu */);
121 }
122 
123 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
124                                     Register dst, Address src, Register tmp1, Register tmp_thread) {
125   bool on_oop = is_reference_type(type);
126   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
127   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
128   bool on_reference = on_weak || on_phantom;
129   ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
130   if (on_oop && on_reference) {
131     Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
132 
133 #ifndef _LP64
134     // Work around the x86_32 bug that only manifests with Loom for some reason.
135     // MacroAssembler::resolve_weak_handle calls this barrier with tmp_thread == noreg.
136     if (thread == noreg) {
137       if (dst != rcx && tmp1 != rcx) {
138         thread = rcx;
139       } else if (dst != rdx && tmp1 != rdx) {
140         thread = rdx;
141       } else if (dst != rdi && tmp1 != rdi) {
142         thread = rdi;
143       }
144     }
145     assert_different_registers(dst, tmp1, thread);
146     __ push(thread);
147     __ get_thread(thread);
148 #endif
149 
150     // Generate the G1 pre-barrier code to log the value of
151     // the referent field in an SATB buffer.
152     g1_write_barrier_pre(masm /* masm */,
153                          noreg /* obj */,
154                          dst /* pre_val */,
155                          thread /* thread */,
156                          tmp1 /* tmp */,
157                          true /* tosca_live */,
158                          true /* expand_call */);
159 
160 #ifndef _LP64
161     __ pop(thread);
162 #endif
163   }
164 }
165 
166 static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
167                                      const Register thread, const Register value, const Register temp) {
168   // This code assumes that buffer index is pointer sized.
169   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
170   // Can we store a value in the given thread's buffer?
171   // (The index field is typed as size_t.)
172   __ movptr(temp, Address(thread, in_bytes(index_offset)));   // temp := *(index address)
173   __ testptr(temp, temp);                                     // index == 0?
174   __ jcc(Assembler::zero, runtime);                           // jump to runtime if index == 0 (full buffer)
175   // The buffer is not full, store value into it.
176   __ subptr(temp, wordSize);                                  // temp := next index
177   __ movptr(Address(thread, in_bytes(index_offset)), temp);   // *(index address) := next index
178   __ addptr(temp, Address(thread, in_bytes(buffer_offset)));  // temp := buffer address + next index
179   __ movptr(Address(temp, 0), value);                         // *(buffer address + next index) := value
180 }
181 
182 static void generate_pre_barrier_fast_path(MacroAssembler* masm,
183                                            const Register thread) {
184   Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
185   // Is marking active?
186   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
187     __ cmpl(in_progress, 0);
188   } else {
189     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
190     __ cmpb(in_progress, 0);
191   }
192 }
193 
194 static void generate_pre_barrier_slow_path(MacroAssembler* masm,
195                                            const Register obj,
196                                            const Register pre_val,
197                                            const Register thread,
198                                            const Register tmp,
199                                            Label& done,
200                                            Label& runtime) {
201   // Do we need to load the previous value?
202   if (obj != noreg) {
203     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
204   }
205   // Is the previous value null?
206   __ cmpptr(pre_val, NULL_WORD);
207   __ jcc(Assembler::equal, done);
208   generate_queue_insertion(masm,
209                            G1ThreadLocalData::satb_mark_queue_index_offset(),
210                            G1ThreadLocalData::satb_mark_queue_buffer_offset(),
211                            runtime,
212                            thread, pre_val, tmp);
213   __ jmp(done);
214 }
215 
216 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
217                                                  Register obj,
218                                                  Register pre_val,
219                                                  Register thread,
220                                                  Register tmp,
221                                                  bool tosca_live,
222                                                  bool expand_call) {
223   // If expand_call is true then we expand the call_VM_leaf macro
224   // directly to skip generating the check by
225   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
226 
227 #ifdef _LP64
228   assert(thread == r15_thread, "must be");
229 #endif // _LP64
230 
231   Label done;
232   Label runtime;
233 
234   assert(pre_val != noreg, "check this code");
235 
236   if (obj != noreg) {
237     assert_different_registers(obj, pre_val, tmp);
238     assert(pre_val != rax, "check this code");
239   }
240 
241   generate_pre_barrier_fast_path(masm, thread);
242   // If marking is not active (*(mark queue active address) == 0), jump to done
243   __ jcc(Assembler::equal, done);
244   generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
245 
246   __ bind(runtime);
247 
248   if (EnableValhalla && InlineTypePassFieldsAsArgs) {
249     // Barriers might be emitted when converting between (scalarized) calling conventions for inline
250     // types. Save all argument registers before calling into the runtime.
251     // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64 )
252     __ pusha();
253     __ subptr(rsp, 64);
254     __ movdbl(Address(rsp, 0),  j_farg0);
255     __ movdbl(Address(rsp, 8),  j_farg1);
256     __ movdbl(Address(rsp, 16), j_farg2);
257     __ movdbl(Address(rsp, 24), j_farg3);
258     __ movdbl(Address(rsp, 32), j_farg4);
259     __ movdbl(Address(rsp, 40), j_farg5);
260     __ movdbl(Address(rsp, 48), j_farg6);
261     __ movdbl(Address(rsp, 56), j_farg7);
262   } else {
263     // Determine and save the live input values
264     __ push_call_clobbered_registers();
265   }
266 
267   // Calling the runtime using the regular call_VM_leaf mechanism generates
268   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
269   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
270   //
271   // If we care generating the pre-barrier without a frame (e.g. in the
272   // intrinsified Reference.get() routine) then ebp might be pointing to
273   // the caller frame and so this check will most likely fail at runtime.
274   //
275   // Expanding the call directly bypasses the generation of the check.
276   // So when we do not have have a full interpreter frame on the stack
277   // expand_call should be passed true.
278 
279   if (expand_call) {
280     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
281 #ifdef _LP64
282     if (c_rarg1 != thread) {
283       __ mov(c_rarg1, thread);
284     }
285     if (c_rarg0 != pre_val) {
286       __ mov(c_rarg0, pre_val);
287     }
288 #else
289     __ push(thread);
290     __ push(pre_val);
291 #endif
292     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
293   } else {
294     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
295   }
296 
297   if (EnableValhalla && InlineTypePassFieldsAsArgs) {
298     // Restore registers
299     __ movdbl(j_farg0, Address(rsp, 0));
300     __ movdbl(j_farg1, Address(rsp, 8));
301     __ movdbl(j_farg2, Address(rsp, 16));
302     __ movdbl(j_farg3, Address(rsp, 24));
303     __ movdbl(j_farg4, Address(rsp, 32));
304     __ movdbl(j_farg5, Address(rsp, 40));
305     __ movdbl(j_farg6, Address(rsp, 48));
306     __ movdbl(j_farg7, Address(rsp, 56));
307     __ addptr(rsp, 64);
308     __ popa();
309   } else {
310     __ pop_call_clobbered_registers();
311   }
312 
313   __ bind(done);
314 }
315 
316 static void generate_post_barrier_fast_path(MacroAssembler* masm,
317                                             const Register store_addr,
318                                             const Register new_val,
319                                             const Register tmp,
320                                             const Register tmp2,
321                                             Label& done,
322                                             bool new_val_may_be_null) {
323   CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
324   // Does store cross heap regions?
325   __ movptr(tmp, store_addr);                                    // tmp := store address
326   __ xorptr(tmp, new_val);                                       // tmp := store address ^ new value
327   __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes);               // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
328   __ jcc(Assembler::equal, done);
329   // Crosses regions, storing null?
330   if (new_val_may_be_null) {
331     __ cmpptr(new_val, NULL_WORD);                               // new value == null?
332     __ jcc(Assembler::equal, done);
333   }
334   // Storing region crossing non-null, is card young?
335   __ movptr(tmp, store_addr);                                    // tmp := store address
336   __ shrptr(tmp, CardTable::card_shift());                       // tmp := card address relative to card table base
337   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
338   // a valid address and therefore is not properly handled by the relocation code.
339   __ movptr(tmp2, (intptr_t)ct->card_table()->byte_map_base());  // tmp2 := card table base address
340   __ addptr(tmp, tmp2);                                          // tmp := card address
341   __ cmpb(Address(tmp, 0), G1CardTable::g1_young_card_val());    // *(card address) == young_card_val?
342 }
343 
344 static void generate_post_barrier_slow_path(MacroAssembler* masm,
345                                             const Register thread,
346                                             const Register tmp,
347                                             const Register tmp2,
348                                             Label& done,
349                                             Label& runtime) {
350   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));  // StoreLoad membar
351   __ cmpb(Address(tmp, 0), G1CardTable::dirty_card_val());       // *(card address) == dirty_card_val?
352   __ jcc(Assembler::equal, done);
353   // Storing a region crossing, non-null oop, card is clean.
354   // Dirty card and log.
355   __ movb(Address(tmp, 0), G1CardTable::dirty_card_val());       // *(card address) := dirty_card_val
356   generate_queue_insertion(masm,
357                            G1ThreadLocalData::dirty_card_queue_index_offset(),
358                            G1ThreadLocalData::dirty_card_queue_buffer_offset(),
359                            runtime,
360                            thread, tmp, tmp2);
361   __ jmp(done);
362 }
363 
364 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
365                                                   Register store_addr,
366                                                   Register new_val,
367                                                   Register thread,
368                                                   Register tmp,
369                                                   Register tmp2) {
370 #ifdef _LP64
371   assert(thread == r15_thread, "must be");
372 #endif // _LP64
373 
374   Label done;
375   Label runtime;
376 
377   generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
378   // If card is young, jump to done
379   __ jcc(Assembler::equal, done);
380   generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
381 
382   __ bind(runtime);
383   // Barriers might be emitted when converting between (scalarized) calling conventions for inline
384   // types. Save all argument registers before calling into the runtime.
385   // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64)
386   __ pusha();
387   __ subptr(rsp, 64);
388   __ movdbl(Address(rsp, 0),  j_farg0);
389   __ movdbl(Address(rsp, 8),  j_farg1);
390   __ movdbl(Address(rsp, 16), j_farg2);
391   __ movdbl(Address(rsp, 24), j_farg3);
392   __ movdbl(Address(rsp, 32), j_farg4);
393   __ movdbl(Address(rsp, 40), j_farg5);
394   __ movdbl(Address(rsp, 48), j_farg6);
395   __ movdbl(Address(rsp, 56), j_farg7);
396 
397   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
398 
399   // Restore registers
400   __ movdbl(j_farg0, Address(rsp, 0));
401   __ movdbl(j_farg1, Address(rsp, 8));
402   __ movdbl(j_farg2, Address(rsp, 16));
403   __ movdbl(j_farg3, Address(rsp, 24));
404   __ movdbl(j_farg4, Address(rsp, 32));
405   __ movdbl(j_farg5, Address(rsp, 40));
406   __ movdbl(j_farg6, Address(rsp, 48));
407   __ movdbl(j_farg7, Address(rsp, 56));
408   __ addptr(rsp, 64);
409   __ popa();
410 
411   __ bind(done);
412 }
413 
414 #if defined(COMPILER2)
415 
416 static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
417 #ifdef _LP64
418   SaveLiveRegisters save_registers(masm, stub);
419   if (c_rarg0 != arg) {
420     __ mov(c_rarg0, arg);
421   }
422   __ mov(c_rarg1, r15_thread);
423   // rax is a caller-saved, non-argument-passing register, so it does not
424   // interfere with c_rarg0 or c_rarg1. If it contained any live value before
425   // entering this stub, it is saved at this point, and restored after the
426   // call. If it did not contain any live value, it is free to be used. In
427   // either case, it is safe to use it here as a call scratch register.
428   __ call(RuntimeAddress(runtime_path), rax);
429 #else
430   Unimplemented();
431 #endif // _LP64
432 }
433 
434 void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
435                                                     Register obj,
436                                                     Register pre_val,
437                                                     Register thread,
438                                                     Register tmp,
439                                                     G1PreBarrierStubC2* stub) {
440 #ifdef _LP64
441   assert(thread == r15_thread, "must be");
442 #endif // _LP64
443   assert(pre_val != noreg, "check this code");
444   if (obj != noreg) {
445     assert_different_registers(obj, pre_val, tmp);
446   }
447 
448   stub->initialize_registers(obj, pre_val, thread, tmp);
449 
450   generate_pre_barrier_fast_path(masm, thread);
451   // If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
452   __ jcc(Assembler::notEqual, *stub->entry());
453 
454   __ bind(*stub->continuation());
455 }
456 
457 void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
458                                                          G1PreBarrierStubC2* stub) const {
459   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
460   Label runtime;
461   Register obj = stub->obj();
462   Register pre_val = stub->pre_val();
463   Register thread = stub->thread();
464   Register tmp = stub->tmp1();
465   assert(stub->tmp2() == noreg, "not needed in this platform");
466 
467   __ bind(*stub->entry());
468   generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, *stub->continuation(), runtime);
469 
470   __ bind(runtime);
471   generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
472   __ jmp(*stub->continuation());
473 }
474 
475 void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
476                                                      Register store_addr,
477                                                      Register new_val,
478                                                      Register thread,
479                                                      Register tmp,
480                                                      Register tmp2,
481                                                      G1PostBarrierStubC2* stub) {
482 #ifdef _LP64
483   assert(thread == r15_thread, "must be");
484 #endif // _LP64
485 
486   stub->initialize_registers(thread, tmp, tmp2);
487 
488   bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
489   generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, *stub->continuation(), new_val_may_be_null);
490   // If card is not young, jump to stub (slow path)
491   __ jcc(Assembler::notEqual, *stub->entry());
492 
493   __ bind(*stub->continuation());
494 }
495 
496 void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
497                                                           G1PostBarrierStubC2* stub) const {
498   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
499   Label runtime;
500   Register thread = stub->thread();
501   Register tmp = stub->tmp1(); // tmp holds the card address.
502   Register tmp2 = stub->tmp2();
503   assert(stub->tmp3() == noreg, "not needed in this platform");
504 
505   __ bind(*stub->entry());
506   generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
507 
508   __ bind(runtime);
509   generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
510   __ jmp(*stub->continuation());
511 }
512 
513 #endif // COMPILER2
514 
515 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
516                                          Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
517   bool in_heap = (decorators & IN_HEAP) != 0;
518   bool as_normal = (decorators & AS_NORMAL) != 0;
519   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
520 
521   bool needs_pre_barrier = as_normal && !dest_uninitialized;
522   bool needs_post_barrier = val != noreg && in_heap;
523 
524   Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
525   // flatten object address if needed
526   // We do it regardless of precise because we need the registers
527   if (dst.index() == noreg && dst.disp() == 0) {
528     if (dst.base() != tmp1) {
529       __ movptr(tmp1, dst.base());
530     }
531   } else {
532     __ lea(tmp1, dst);
533   }
534 
535 #ifndef _LP64
536   InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
537 #endif
538 
539   NOT_LP64(__ get_thread(rcx));
540   NOT_LP64(imasm->save_bcp());
541 
542   if (needs_pre_barrier) {
543     g1_write_barrier_pre(masm /*masm*/,
544                          tmp1 /* obj */,
545                          tmp2 /* pre_val */,
546                          rthread /* thread */,
547                          tmp3  /* tmp */,
548                          val != noreg /* tosca_live */,
549                          false /* expand_call */);
550   }
551   if (val == noreg) {
552     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
553   } else {
554     Register new_val = val;
555     if (needs_post_barrier) {
556       // G1 barrier needs uncompressed oop for region cross check.
557       if (UseCompressedOops) {
558         new_val = tmp2;
559         __ movptr(new_val, val);
560       }
561     }
562     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
563     if (needs_post_barrier) {
564       g1_write_barrier_post(masm /*masm*/,
565                             tmp1 /* store_adr */,
566                             new_val /* new_val */,
567                             rthread /* thread */,
568                             tmp3 /* tmp */,
569                             tmp2 /* tmp2 */);
570     }
571   }
572   NOT_LP64(imasm->restore_bcp());
573 }
574 
575 #ifdef COMPILER1
576 
577 #undef __
578 #define __ ce->masm()->
579 
580 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
581   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
582   // At this point we know that marking is in progress.
583   // If do_load() is true then we have to emit the
584   // load of the previous value; otherwise it has already
585   // been loaded into _pre_val.
586 
587   __ bind(*stub->entry());
588   assert(stub->pre_val()->is_register(), "Precondition.");
589 
590   Register pre_val_reg = stub->pre_val()->as_register();
591 
592   if (stub->do_load()) {
593     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
594   }
595 
596   __ cmpptr(pre_val_reg, NULL_WORD);
597   __ jcc(Assembler::equal, *stub->continuation());
598   ce->store_parameter(stub->pre_val()->as_register(), 0);
599   __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
600   __ jmp(*stub->continuation());
601 
602 }
603 
604 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
605   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
606   __ bind(*stub->entry());
607   assert(stub->addr()->is_register(), "Precondition.");
608   assert(stub->new_val()->is_register(), "Precondition.");
609   Register new_val_reg = stub->new_val()->as_register();
610   __ cmpptr(new_val_reg, NULL_WORD);
611   __ jcc(Assembler::equal, *stub->continuation());
612   ce->store_parameter(stub->addr()->as_pointer_register(), 0);
613   __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
614   __ jmp(*stub->continuation());
615 }
616 
617 #undef __
618 
619 #define __ sasm->
620 
621 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
622   // Generated code assumes that buffer index is pointer sized.
623   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
624 
625   __ prologue("g1_pre_barrier", false);
626   // arg0 : previous value of memory
627 
628   __ push(rax);
629   __ push(rdx);
630 
631   const Register pre_val = rax;
632   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
633   const Register tmp = rdx;
634 
635   NOT_LP64(__ get_thread(thread);)
636 
637   Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
638   Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
639   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
640 
641   Label done;
642   Label runtime;
643 
644   // Is marking still active?
645   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
646     __ cmpl(queue_active, 0);
647   } else {
648     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
649     __ cmpb(queue_active, 0);
650   }
651   __ jcc(Assembler::equal, done);
652 
653   // Can we store original value in the thread's buffer?
654 
655   __ movptr(tmp, queue_index);
656   __ testptr(tmp, tmp);
657   __ jcc(Assembler::zero, runtime);
658   __ subptr(tmp, wordSize);
659   __ movptr(queue_index, tmp);
660   __ addptr(tmp, buffer);
661 
662   // prev_val (rax)
663   __ load_parameter(0, pre_val);
664   __ movptr(Address(tmp, 0), pre_val);
665   __ jmp(done);
666 
667   __ bind(runtime);
668 
669   __ push_call_clobbered_registers();
670 
671   // load the pre-value
672   __ load_parameter(0, rcx);
673   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread);
674 
675   __ pop_call_clobbered_registers();
676 
677   __ bind(done);
678 
679   __ pop(rdx);
680   __ pop(rax);
681 
682   __ epilogue();
683 }
684 
685 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
686   __ prologue("g1_post_barrier", false);
687 
688   CardTableBarrierSet* ct =
689     barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
690 
691   Label done;
692   Label enqueued;
693   Label runtime;
694 
695   // At this point we know new_value is non-null and the new_value crosses regions.
696   // Must check to see if card is already dirty
697 
698   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
699 
700   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
701   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
702 
703   __ push(rax);
704   __ push(rcx);
705 
706   const Register cardtable = rax;
707   const Register card_addr = rcx;
708 
709   __ load_parameter(0, card_addr);
710   __ shrptr(card_addr, CardTable::card_shift());
711   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
712   // a valid address and therefore is not properly handled by the relocation code.
713   __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
714   __ addptr(card_addr, cardtable);
715 
716   NOT_LP64(__ get_thread(thread);)
717 
718   __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
719   __ jcc(Assembler::equal, done);
720 
721   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
722   __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
723   __ jcc(Assembler::equal, done);
724 
725   // storing region crossing non-null, card is clean.
726   // dirty card and log.
727 
728   __ movb(Address(card_addr, 0), CardTable::dirty_card_val());
729 
730   const Register tmp = rdx;
731   __ push(rdx);
732 
733   __ movptr(tmp, queue_index);
734   __ testptr(tmp, tmp);
735   __ jcc(Assembler::zero, runtime);
736   __ subptr(tmp, wordSize);
737   __ movptr(queue_index, tmp);
738   __ addptr(tmp, buffer);
739   __ movptr(Address(tmp, 0), card_addr);
740   __ jmp(enqueued);
741 
742   __ bind(runtime);
743   __ push_call_clobbered_registers();
744 
745   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
746 
747   __ pop_call_clobbered_registers();
748 
749   __ bind(enqueued);
750   __ pop(rdx);
751 
752   __ bind(done);
753   __ pop(rcx);
754   __ pop(rax);
755 
756   __ epilogue();
757 }
758 
759 #undef __
760 
761 #endif // COMPILER1
--- EOF ---