1 /*
  2  * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "asm/macroAssembler.inline.hpp"
 27 #include "gc/g1/g1BarrierSet.hpp"
 28 #include "gc/g1/g1BarrierSetAssembler.hpp"
 29 #include "gc/g1/g1BarrierSetRuntime.hpp"
 30 #include "gc/g1/g1CardTable.hpp"
 31 #include "gc/g1/g1ThreadLocalData.hpp"
 32 #include "gc/g1/heapRegion.hpp"
 33 #include "interpreter/interp_masm.hpp"
 34 #include "runtime/sharedRuntime.hpp"
 35 #include "utilities/debug.hpp"
 36 #include "utilities/macros.hpp"
 37 #ifdef COMPILER1
 38 #include "c1/c1_LIRAssembler.hpp"
 39 #include "c1/c1_MacroAssembler.hpp"
 40 #include "gc/g1/c1/g1BarrierSetC1.hpp"
 41 #endif
 42 
 43 #define __ masm->
 44 
 45 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
 46                                                             Register addr, Register count) {
 47   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 48 
 49   if (!dest_uninitialized) {
 50     Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
 51 #ifndef _LP64
 52     __ push(thread);
 53     __ get_thread(thread);
 54 #endif
 55 
 56     Label filtered;
 57     Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
 58     // Is marking active?
 59     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 60       __ cmpl(in_progress, 0);
 61     } else {
 62       assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 63       __ cmpb(in_progress, 0);
 64     }
 65 
 66     NOT_LP64(__ pop(thread);)
 67 
 68     __ jcc(Assembler::equal, filtered);
 69 
 70     __ push_call_clobbered_registers(false /* save_fpu */);
 71 #ifdef _LP64
 72     if (count == c_rarg0) {
 73       if (addr == c_rarg1) {
 74         // exactly backwards!!
 75         __ xchgptr(c_rarg1, c_rarg0);
 76       } else {
 77         __ movptr(c_rarg1, count);
 78         __ movptr(c_rarg0, addr);
 79       }
 80     } else {
 81       __ movptr(c_rarg0, addr);
 82       __ movptr(c_rarg1, count);
 83     }
 84     if (UseCompressedOops) {
 85       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
 86     } else {
 87       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
 88     }
 89 #else
 90     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry),
 91                     addr, count);
 92 #endif
 93     __ pop_call_clobbered_registers(false /* save_fpu */);
 94 
 95     __ bind(filtered);
 96   }
 97 }
 98 
 99 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
100                                                              Register addr, Register count, Register tmp) {
101   __ push_call_clobbered_registers(false /* save_fpu */);
102 #ifdef _LP64
103   if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
104     assert_different_registers(c_rarg1, addr);
105     __ mov(c_rarg1, count);
106     __ mov(c_rarg0, addr);
107   } else {
108     assert_different_registers(c_rarg0, count);
109     __ mov(c_rarg0, addr);
110     __ mov(c_rarg1, count);
111   }
112   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
113 #else
114   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry),
115                   addr, count);
116 #endif
117   __ pop_call_clobbered_registers(false /* save_fpu */);
118 }
119 
120 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
121                                     Register dst, Address src, Register tmp1, Register tmp_thread) {
122   bool on_oop = is_reference_type(type);
123   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
124   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
125   bool on_reference = on_weak || on_phantom;
126   ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
127   if (on_oop && on_reference) {
128     Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
129 
130 #ifndef _LP64
131     // Work around the x86_32 bug that only manifests with Loom for some reason.
132     // MacroAssembler::resolve_weak_handle calls this barrier with tmp_thread == noreg.
133     if (thread == noreg) {
134       if (dst != rcx && tmp1 != rcx) {
135         thread = rcx;
136       } else if (dst != rdx && tmp1 != rdx) {
137         thread = rdx;
138       } else if (dst != rdi && tmp1 != rdi) {
139         thread = rdi;
140       }
141     }
142     assert_different_registers(dst, tmp1, thread);
143     __ push(thread);
144     __ get_thread(thread);
145 #endif
146 
147     // Generate the G1 pre-barrier code to log the value of
148     // the referent field in an SATB buffer.
149     g1_write_barrier_pre(masm /* masm */,
150                          noreg /* obj */,
151                          dst /* pre_val */,
152                          thread /* thread */,
153                          tmp1 /* tmp */,
154                          true /* tosca_live */,
155                          true /* expand_call */);
156 
157 #ifndef _LP64
158     __ pop(thread);
159 #endif
160   }
161 }
162 
163 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
164                                                  Register obj,
165                                                  Register pre_val,
166                                                  Register thread,
167                                                  Register tmp,
168                                                  bool tosca_live,
169                                                  bool expand_call) {
170   // If expand_call is true then we expand the call_VM_leaf macro
171   // directly to skip generating the check by
172   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
173 
174 #ifdef _LP64
175   assert(thread == r15_thread, "must be");
176 #endif // _LP64
177 
178   Label done;
179   Label runtime;
180 
181   assert(pre_val != noreg, "check this code");
182 
183   if (obj != noreg) {
184     assert_different_registers(obj, pre_val, tmp);
185     assert(pre_val != rax, "check this code");
186   }
187 
188   Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
189   Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
190   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
191 
192   // Is marking active?
193   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
194     __ cmpl(in_progress, 0);
195   } else {
196     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
197     __ cmpb(in_progress, 0);
198   }
199   __ jcc(Assembler::equal, done);
200 
201   // Do we need to load the previous value?
202   if (obj != noreg) {
203     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
204   }
205 
206   // Is the previous value null?
207   __ cmpptr(pre_val, NULL_WORD);
208   __ jcc(Assembler::equal, done);
209 
210   // Can we store original value in the thread's buffer?
211   // Is index == 0?
212   // (The index field is typed as size_t.)
213   __ movptr(tmp, index);                   // tmp := *index_adr
214   __ cmpptr(tmp, 0);                       // tmp == 0?
215   __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
216 
217   __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
218   __ movptr(index, tmp);                   // *index_adr := tmp
219   __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
220 
221   // Record the previous value
222   __ movptr(Address(tmp, 0), pre_val);
223   __ jmp(done);
224 
225   __ bind(runtime);
226 
227   if (EnableValhalla && InlineTypePassFieldsAsArgs) {
228     // Barriers might be emitted when converting between (scalarized) calling conventions for inline
229     // types. Save all argument registers before calling into the runtime.
230     // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64 )
231     __ pusha();
232     __ subptr(rsp, 64);
233     __ movdbl(Address(rsp, 0),  j_farg0);
234     __ movdbl(Address(rsp, 8),  j_farg1);
235     __ movdbl(Address(rsp, 16), j_farg2);
236     __ movdbl(Address(rsp, 24), j_farg3);
237     __ movdbl(Address(rsp, 32), j_farg4);
238     __ movdbl(Address(rsp, 40), j_farg5);
239     __ movdbl(Address(rsp, 48), j_farg6);
240     __ movdbl(Address(rsp, 56), j_farg7);
241   } else {
242     // Determine and save the live input values
243     __ push_call_clobbered_registers();
244   }
245 
246   // Calling the runtime using the regular call_VM_leaf mechanism generates
247   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
248   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
249   //
250   // If we care generating the pre-barrier without a frame (e.g. in the
251   // intrinsified Reference.get() routine) then ebp might be pointing to
252   // the caller frame and so this check will most likely fail at runtime.
253   //
254   // Expanding the call directly bypasses the generation of the check.
255   // So when we do not have have a full interpreter frame on the stack
256   // expand_call should be passed true.
257 
258   if (expand_call) {
259     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
260 #ifdef _LP64
261     if (c_rarg1 != thread) {
262       __ mov(c_rarg1, thread);
263     }
264     if (c_rarg0 != pre_val) {
265       __ mov(c_rarg0, pre_val);
266     }
267 #else
268     __ push(thread);
269     __ push(pre_val);
270 #endif
271     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
272   } else {
273     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
274   }
275 
276   if (EnableValhalla && InlineTypePassFieldsAsArgs) {
277     // Restore registers
278     __ movdbl(j_farg0, Address(rsp, 0));
279     __ movdbl(j_farg1, Address(rsp, 8));
280     __ movdbl(j_farg2, Address(rsp, 16));
281     __ movdbl(j_farg3, Address(rsp, 24));
282     __ movdbl(j_farg4, Address(rsp, 32));
283     __ movdbl(j_farg5, Address(rsp, 40));
284     __ movdbl(j_farg6, Address(rsp, 48));
285     __ movdbl(j_farg7, Address(rsp, 56));
286     __ addptr(rsp, 64);
287     __ popa();
288   } else {
289     __ pop_call_clobbered_registers();
290   }
291 
292   __ bind(done);
293 }
294 
295 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
296                                                   Register store_addr,
297                                                   Register new_val,
298                                                   Register thread,
299                                                   Register tmp,
300                                                   Register tmp2) {
301   // Generated code assumes that buffer index is pointer sized.
302   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
303 #ifdef _LP64
304   assert(thread == r15_thread, "must be");
305 #endif // _LP64
306 
307   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
308   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
309 
310   CardTableBarrierSet* ct =
311     barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
312 
313   Label done;
314   Label runtime;
315 
316   // Does store cross heap regions?
317 
318   __ movptr(tmp, store_addr);
319   __ xorptr(tmp, new_val);
320   __ shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
321   __ jcc(Assembler::equal, done);
322 
323   // crosses regions, storing NULL?
324 
325   __ cmpptr(new_val, NULL_WORD);
326   __ jcc(Assembler::equal, done);
327 
328   // storing region crossing non-NULL, is card already dirty?
329 
330   const Register card_addr = tmp;
331   const Register cardtable = tmp2;
332 
333   __ movptr(card_addr, store_addr);
334   __ shrptr(card_addr, CardTable::card_shift());
335   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
336   // a valid address and therefore is not properly handled by the relocation code.
337   __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
338   __ addptr(card_addr, cardtable);
339 
340   __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
341   __ jcc(Assembler::equal, done);
342 
343   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
344   __ cmpb(Address(card_addr, 0), G1CardTable::dirty_card_val());
345   __ jcc(Assembler::equal, done);
346 
347 
348   // storing a region crossing, non-NULL oop, card is clean.
349   // dirty card and log.
350 
351   __ movb(Address(card_addr, 0), G1CardTable::dirty_card_val());
352 
353   __ movptr(tmp2, queue_index);
354   __ testptr(tmp2, tmp2);
355   __ jcc(Assembler::zero, runtime);
356   __ subptr(tmp2, wordSize);
357   __ movptr(queue_index, tmp2);
358   __ addptr(tmp2, buffer);
359   __ movptr(Address(tmp2, 0), card_addr);
360   __ jmp(done);
361 
362   __ bind(runtime);
363   // Barriers might be emitted when converting between (scalarized) calling conventions for inline
364   // types. Save all argument registers before calling into the runtime.
365   // TODO: use push_set() (see JDK-8283327 push/pop_call_clobbered_registers & aarch64)
366   __ pusha();
367   __ subptr(rsp, 64);
368   __ movdbl(Address(rsp, 0),  j_farg0);
369   __ movdbl(Address(rsp, 8),  j_farg1);
370   __ movdbl(Address(rsp, 16), j_farg2);
371   __ movdbl(Address(rsp, 24), j_farg3);
372   __ movdbl(Address(rsp, 32), j_farg4);
373   __ movdbl(Address(rsp, 40), j_farg5);
374   __ movdbl(Address(rsp, 48), j_farg6);
375   __ movdbl(Address(rsp, 56), j_farg7);
376 
377 #ifdef _LP64
378   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, r15_thread);
379 #else
380   __ push(thread);
381   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
382   __ pop(thread);
383 #endif
384 
385   // Restore registers
386   __ movdbl(j_farg0, Address(rsp, 0));
387   __ movdbl(j_farg1, Address(rsp, 8));
388   __ movdbl(j_farg2, Address(rsp, 16));
389   __ movdbl(j_farg3, Address(rsp, 24));
390   __ movdbl(j_farg4, Address(rsp, 32));
391   __ movdbl(j_farg5, Address(rsp, 40));
392   __ movdbl(j_farg6, Address(rsp, 48));
393   __ movdbl(j_farg7, Address(rsp, 56));
394   __ addptr(rsp, 64);
395   __ popa();
396 
397   __ bind(done);
398 }
399 
400 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
401                                          Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
402   bool in_heap = (decorators & IN_HEAP) != 0;
403   bool as_normal = (decorators & AS_NORMAL) != 0;
404   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
405 
406   bool needs_pre_barrier = as_normal && !dest_uninitialized;
407   bool needs_post_barrier = val != noreg && in_heap;
408 
409   Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
410   // flatten object address if needed
411   // We do it regardless of precise because we need the registers
412   if (dst.index() == noreg && dst.disp() == 0) {
413     if (dst.base() != tmp1) {
414       __ movptr(tmp1, dst.base());
415     }
416   } else {
417     __ lea(tmp1, dst);
418   }
419 
420 #ifndef _LP64
421   InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
422 #endif
423 
424   NOT_LP64(__ get_thread(rcx));
425   NOT_LP64(imasm->save_bcp());
426 
427   if (needs_pre_barrier) {
428     g1_write_barrier_pre(masm /*masm*/,
429                          tmp1 /* obj */,
430                          tmp2 /* pre_val */,
431                          rthread /* thread */,
432                          tmp3  /* tmp */,
433                          val != noreg /* tosca_live */,
434                          false /* expand_call */);
435   }
436   if (val == noreg) {
437     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
438   } else {
439     Register new_val = val;
440     if (needs_post_barrier) {
441       // G1 barrier needs uncompressed oop for region cross check.
442       if (UseCompressedOops) {
443         new_val = tmp2;
444         __ movptr(new_val, val);
445       }
446     }
447     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
448     if (needs_post_barrier) {
449       g1_write_barrier_post(masm /*masm*/,
450                             tmp1 /* store_adr */,
451                             new_val /* new_val */,
452                             rthread /* thread */,
453                             tmp3 /* tmp */,
454                             tmp2 /* tmp2 */);
455     }
456   }
457   NOT_LP64(imasm->restore_bcp());
458 }
459 
460 #ifdef COMPILER1
461 
462 #undef __
463 #define __ ce->masm()->
464 
465 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
466   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
467   // At this point we know that marking is in progress.
468   // If do_load() is true then we have to emit the
469   // load of the previous value; otherwise it has already
470   // been loaded into _pre_val.
471 
472   __ bind(*stub->entry());
473   assert(stub->pre_val()->is_register(), "Precondition.");
474 
475   Register pre_val_reg = stub->pre_val()->as_register();
476 
477   if (stub->do_load()) {
478     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
479   }
480 
481   __ cmpptr(pre_val_reg, NULL_WORD);
482   __ jcc(Assembler::equal, *stub->continuation());
483   ce->store_parameter(stub->pre_val()->as_register(), 0);
484   __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
485   __ jmp(*stub->continuation());
486 
487 }
488 
489 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
490   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
491   __ bind(*stub->entry());
492   assert(stub->addr()->is_register(), "Precondition.");
493   assert(stub->new_val()->is_register(), "Precondition.");
494   Register new_val_reg = stub->new_val()->as_register();
495   __ cmpptr(new_val_reg, NULL_WORD);
496   __ jcc(Assembler::equal, *stub->continuation());
497   ce->store_parameter(stub->addr()->as_pointer_register(), 0);
498   __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
499   __ jmp(*stub->continuation());
500 }
501 
502 #undef __
503 
504 #define __ sasm->
505 
506 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
507   // Generated code assumes that buffer index is pointer sized.
508   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
509 
510   __ prologue("g1_pre_barrier", false);
511   // arg0 : previous value of memory
512 
513   __ push(rax);
514   __ push(rdx);
515 
516   const Register pre_val = rax;
517   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
518   const Register tmp = rdx;
519 
520   NOT_LP64(__ get_thread(thread);)
521 
522   Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
523   Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
524   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
525 
526   Label done;
527   Label runtime;
528 
529   // Is marking still active?
530   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
531     __ cmpl(queue_active, 0);
532   } else {
533     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
534     __ cmpb(queue_active, 0);
535   }
536   __ jcc(Assembler::equal, done);
537 
538   // Can we store original value in the thread's buffer?
539 
540   __ movptr(tmp, queue_index);
541   __ testptr(tmp, tmp);
542   __ jcc(Assembler::zero, runtime);
543   __ subptr(tmp, wordSize);
544   __ movptr(queue_index, tmp);
545   __ addptr(tmp, buffer);
546 
547   // prev_val (rax)
548   __ load_parameter(0, pre_val);
549   __ movptr(Address(tmp, 0), pre_val);
550   __ jmp(done);
551 
552   __ bind(runtime);
553 
554   __ push_call_clobbered_registers();
555 
556   // load the pre-value
557   __ load_parameter(0, rcx);
558   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread);
559 
560   __ pop_call_clobbered_registers();
561 
562   __ bind(done);
563 
564   __ pop(rdx);
565   __ pop(rax);
566 
567   __ epilogue();
568 }
569 
570 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
571   __ prologue("g1_post_barrier", false);
572 
573   CardTableBarrierSet* ct =
574     barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
575 
576   Label done;
577   Label enqueued;
578   Label runtime;
579 
580   // At this point we know new_value is non-NULL and the new_value crosses regions.
581   // Must check to see if card is already dirty
582 
583   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
584 
585   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
586   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
587 
588   __ push(rax);
589   __ push(rcx);
590 
591   const Register cardtable = rax;
592   const Register card_addr = rcx;
593 
594   __ load_parameter(0, card_addr);
595   __ shrptr(card_addr, CardTable::card_shift());
596   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
597   // a valid address and therefore is not properly handled by the relocation code.
598   __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
599   __ addptr(card_addr, cardtable);
600 
601   NOT_LP64(__ get_thread(thread);)
602 
603   __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
604   __ jcc(Assembler::equal, done);
605 
606   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
607   __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
608   __ jcc(Assembler::equal, done);
609 
610   // storing region crossing non-NULL, card is clean.
611   // dirty card and log.
612 
613   __ movb(Address(card_addr, 0), CardTable::dirty_card_val());
614 
615   const Register tmp = rdx;
616   __ push(rdx);
617 
618   __ movptr(tmp, queue_index);
619   __ testptr(tmp, tmp);
620   __ jcc(Assembler::zero, runtime);
621   __ subptr(tmp, wordSize);
622   __ movptr(queue_index, tmp);
623   __ addptr(tmp, buffer);
624   __ movptr(Address(tmp, 0), card_addr);
625   __ jmp(enqueued);
626 
627   __ bind(runtime);
628   __ push_call_clobbered_registers();
629 
630   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
631 
632   __ pop_call_clobbered_registers();
633 
634   __ bind(enqueued);
635   __ pop(rdx);
636 
637   __ bind(done);
638   __ pop(rcx);
639   __ pop(rax);
640 
641   __ epilogue();
642 }
643 
644 #undef __
645 
646 #endif // COMPILER1