1 /*
  2  * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "asm/macroAssembler.inline.hpp"
 27 #include "gc/g1/g1BarrierSet.hpp"
 28 #include "gc/g1/g1BarrierSetAssembler.hpp"
 29 #include "gc/g1/g1BarrierSetRuntime.hpp"
 30 #include "gc/g1/g1CardTable.hpp"
 31 #include "gc/g1/g1ThreadLocalData.hpp"
 32 #include "gc/g1/heapRegion.hpp"
 33 #include "interpreter/interp_masm.hpp"
 34 #include "runtime/sharedRuntime.hpp"
 35 #include "utilities/debug.hpp"
 36 #include "utilities/macros.hpp"
 37 #ifdef COMPILER1
 38 #include "c1/c1_LIRAssembler.hpp"
 39 #include "c1/c1_MacroAssembler.hpp"
 40 #include "gc/g1/c1/g1BarrierSetC1.hpp"
 41 #endif
 42 
 43 #define __ masm->
 44 
 45 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
 46                                                             Register addr, Register count) {
 47   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 48 
 49   if (!dest_uninitialized) {
 50     Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
 51 #ifndef _LP64
 52     __ push(thread);
 53     __ get_thread(thread);
 54 #endif
 55 
 56     Label filtered;
 57     Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
 58     // Is marking active?
 59     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 60       __ cmpl(in_progress, 0);
 61     } else {
 62       assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 63       __ cmpb(in_progress, 0);
 64     }
 65 
 66     NOT_LP64(__ pop(thread);)
 67 
 68     __ jcc(Assembler::equal, filtered);
 69 
 70     __ pusha();                      // push registers
 71 #ifdef _LP64
 72     if (count == c_rarg0) {
 73       if (addr == c_rarg1) {
 74         // exactly backwards!!
 75         __ xchgptr(c_rarg1, c_rarg0);
 76       } else {
 77         __ movptr(c_rarg1, count);
 78         __ movptr(c_rarg0, addr);
 79       }
 80     } else {
 81       __ movptr(c_rarg0, addr);
 82       __ movptr(c_rarg1, count);
 83     }
 84     if (UseCompressedOops) {
 85       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
 86     } else {
 87       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
 88     }
 89 #else
 90     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry),
 91                     addr, count);
 92 #endif
 93     __ popa();
 94 
 95     __ bind(filtered);
 96   }
 97 }
 98 
 99 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
100                                                              Register addr, Register count, Register tmp) {
101   __ pusha();             // push registers (overkill)
102 #ifdef _LP64
103   if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
104     assert_different_registers(c_rarg1, addr);
105     __ mov(c_rarg1, count);
106     __ mov(c_rarg0, addr);
107   } else {
108     assert_different_registers(c_rarg0, count);
109     __ mov(c_rarg0, addr);
110     __ mov(c_rarg1, count);
111   }
112   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
113 #else
114   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry),
115                   addr, count);
116 #endif
117   __ popa();
118 }
119 
120 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
121                                     Register dst, Address src, Register tmp1, Register tmp_thread) {
122   bool on_oop = is_reference_type(type);
123   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
124   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
125   bool on_reference = on_weak || on_phantom;
126   ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
127   if (on_oop && on_reference) {
128     const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
129     NOT_LP64(__ get_thread(thread));
130 
131     // Generate the G1 pre-barrier code to log the value of
132     // the referent field in an SATB buffer.
133     g1_write_barrier_pre(masm /* masm */,
134                          noreg /* obj */,
135                          dst /* pre_val */,
136                          thread /* thread */,
137                          tmp1 /* tmp */,
138                          true /* tosca_live */,
139                          true /* expand_call */);
140   }
141 }
142 
143 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
144                                                  Register obj,
145                                                  Register pre_val,
146                                                  Register thread,
147                                                  Register tmp,
148                                                  bool tosca_live,
149                                                  bool expand_call) {
150   // If expand_call is true then we expand the call_VM_leaf macro
151   // directly to skip generating the check by
152   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
153 
154 #ifdef _LP64
155   assert(thread == r15_thread, "must be");
156 #endif // _LP64
157 
158   Label done;
159   Label runtime;
160 
161   assert(pre_val != noreg, "check this code");
162 
163   if (obj != noreg) {
164     assert_different_registers(obj, pre_val, tmp);
165     assert(pre_val != rax, "check this code");
166   }
167 
168   Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
169   Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
170   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
171 
172   // Is marking active?
173   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
174     __ cmpl(in_progress, 0);
175   } else {
176     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
177     __ cmpb(in_progress, 0);
178   }
179   __ jcc(Assembler::equal, done);
180 
181   // Do we need to load the previous value?
182   if (obj != noreg) {
183     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
184   }
185 
186   // Is the previous value null?
187   __ cmpptr(pre_val, (int32_t) NULL_WORD);
188   __ jcc(Assembler::equal, done);
189 
190   // Can we store original value in the thread's buffer?
191   // Is index == 0?
192   // (The index field is typed as size_t.)
193   __ movptr(tmp, index);                   // tmp := *index_adr
194   __ cmpptr(tmp, 0);                       // tmp == 0?
195   __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
196 
197   __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
198   __ movptr(index, tmp);                   // *index_adr := tmp
199   __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
200 
201   // Record the previous value
202   __ movptr(Address(tmp, 0), pre_val);
203   __ jmp(done);
204 
205   __ bind(runtime);
206   // Barriers might be emitted when converting between (scalarized) calling conventions for inline
207   // types. Save all argument registers before calling into the runtime.
208   __ pusha();
209   __ subptr(rsp, 64);
210   __ movdbl(Address(rsp, 0),  j_farg0);
211   __ movdbl(Address(rsp, 8),  j_farg1);
212   __ movdbl(Address(rsp, 16), j_farg2);
213   __ movdbl(Address(rsp, 24), j_farg3);
214   __ movdbl(Address(rsp, 32), j_farg4);
215   __ movdbl(Address(rsp, 40), j_farg5);
216   __ movdbl(Address(rsp, 48), j_farg6);
217   __ movdbl(Address(rsp, 56), j_farg7);
218 
219   // Calling the runtime using the regular call_VM_leaf mechanism generates
220   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
221   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
222   //
223   // If we care generating the pre-barrier without a frame (e.g. in the
224   // intrinsified Reference.get() routine) then ebp might be pointing to
225   // the caller frame and so this check will most likely fail at runtime.
226   //
227   // Expanding the call directly bypasses the generation of the check.
228   // So when we do not have have a full interpreter frame on the stack
229   // expand_call should be passed true.
230 
231   if (expand_call) {
232     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
233 #ifdef _LP64
234     if (c_rarg1 != thread) {
235       __ mov(c_rarg1, thread);
236     }
237     if (c_rarg0 != pre_val) {
238       __ mov(c_rarg0, pre_val);
239     }
240 #else
241     __ push(thread);
242     __ push(pre_val);
243 #endif
244     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
245   } else {
246     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
247   }
248 
249   // Restore registers
250   __ movdbl(j_farg0, Address(rsp, 0));
251   __ movdbl(j_farg1, Address(rsp, 8));
252   __ movdbl(j_farg2, Address(rsp, 16));
253   __ movdbl(j_farg3, Address(rsp, 24));
254   __ movdbl(j_farg4, Address(rsp, 32));
255   __ movdbl(j_farg5, Address(rsp, 40));
256   __ movdbl(j_farg6, Address(rsp, 48));
257   __ movdbl(j_farg7, Address(rsp, 56));
258   __ addptr(rsp, 64);
259   __ popa();
260 
261   __ bind(done);
262 }
263 
264 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
265                                                   Register store_addr,
266                                                   Register new_val,
267                                                   Register thread,
268                                                   Register tmp,
269                                                   Register tmp2) {
270   // Generated code assumes that buffer index is pointer sized.
271   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
272 #ifdef _LP64
273   assert(thread == r15_thread, "must be");
274 #endif // _LP64
275 
276   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
277   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
278 
279   CardTableBarrierSet* ct =
280     barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
281 
282   Label done;
283   Label runtime;
284 
285   // Does store cross heap regions?
286 
287   __ movptr(tmp, store_addr);
288   __ xorptr(tmp, new_val);
289   __ shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
290   __ jcc(Assembler::equal, done);
291 
292   // crosses regions, storing NULL?
293 
294   __ cmpptr(new_val, (int32_t) NULL_WORD);
295   __ jcc(Assembler::equal, done);
296 
297   // storing region crossing non-NULL, is card already dirty?
298 
299   const Register card_addr = tmp;
300   const Register cardtable = tmp2;
301 
302   __ movptr(card_addr, store_addr);
303   __ shrptr(card_addr, CardTable::card_shift);
304   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
305   // a valid address and therefore is not properly handled by the relocation code.
306   __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
307   __ addptr(card_addr, cardtable);
308 
309   __ cmpb(Address(card_addr, 0), (int)G1CardTable::g1_young_card_val());
310   __ jcc(Assembler::equal, done);
311 
312   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
313   __ cmpb(Address(card_addr, 0), (int)G1CardTable::dirty_card_val());
314   __ jcc(Assembler::equal, done);
315 
316 
317   // storing a region crossing, non-NULL oop, card is clean.
318   // dirty card and log.
319 
320   __ movb(Address(card_addr, 0), (int)G1CardTable::dirty_card_val());
321 
322   __ movptr(tmp2, queue_index);
323   __ testptr(tmp2, tmp2);
324   __ jcc(Assembler::zero, runtime);
325   __ subptr(tmp2, wordSize);
326   __ movptr(queue_index, tmp2);
327   __ addptr(tmp2, buffer);
328   __ movptr(Address(tmp2, 0), card_addr);
329   __ jmp(done);
330 
331   __ bind(runtime);
332   // Barriers might be emitted when converting between (scalarized) calling conventions for inline
333   // types. Save all argument registers before calling into the runtime.
334   __ pusha();
335   __ subptr(rsp, 64);
336   __ movdbl(Address(rsp, 0),  j_farg0);
337   __ movdbl(Address(rsp, 8),  j_farg1);
338   __ movdbl(Address(rsp, 16), j_farg2);
339   __ movdbl(Address(rsp, 24), j_farg3);
340   __ movdbl(Address(rsp, 32), j_farg4);
341   __ movdbl(Address(rsp, 40), j_farg5);
342   __ movdbl(Address(rsp, 48), j_farg6);
343   __ movdbl(Address(rsp, 56), j_farg7);
344 
345 #ifdef _LP64
346   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, r15_thread);
347 #else
348   __ push(thread);
349   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
350   __ pop(thread);
351 #endif
352 
353   // Restore registers
354   __ movdbl(j_farg0, Address(rsp, 0));
355   __ movdbl(j_farg1, Address(rsp, 8));
356   __ movdbl(j_farg2, Address(rsp, 16));
357   __ movdbl(j_farg3, Address(rsp, 24));
358   __ movdbl(j_farg4, Address(rsp, 32));
359   __ movdbl(j_farg5, Address(rsp, 40));
360   __ movdbl(j_farg6, Address(rsp, 48));
361   __ movdbl(j_farg7, Address(rsp, 56));
362   __ addptr(rsp, 64);
363   __ popa();
364 
365   __ bind(done);
366 }
367 
368 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
369                                          Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
370   bool in_heap = (decorators & IN_HEAP) != 0;
371   bool as_normal = (decorators & AS_NORMAL) != 0;
372   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
373 
374   bool needs_pre_barrier = as_normal && !dest_uninitialized;
375   bool needs_post_barrier = val != noreg && in_heap;
376 
377   if (tmp3 == noreg) {
378     tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
379   }
380   Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
381   // flatten object address if needed
382   // We do it regardless of precise because we need the registers
383   if (dst.index() == noreg && dst.disp() == 0) {
384     if (dst.base() != tmp1) {
385       __ movptr(tmp1, dst.base());
386     }
387   } else {
388     __ lea(tmp1, dst);
389   }
390 
391 #ifndef _LP64
392   InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
393 #endif
394 
395   NOT_LP64(__ get_thread(rcx));
396   NOT_LP64(imasm->save_bcp());
397 
398   if (needs_pre_barrier) {
399     g1_write_barrier_pre(masm /*masm*/,
400                          tmp1 /* obj */,
401                          tmp2 /* pre_val */,
402                          rthread /* thread */,
403                          tmp3  /* tmp */,
404                          val != noreg /* tosca_live */,
405                          false /* expand_call */);
406   }
407   if (val == noreg) {
408     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
409   } else {
410     Register new_val = val;
411     if (needs_post_barrier) {
412       // G1 barrier needs uncompressed oop for region cross check.
413       if (UseCompressedOops) {
414         new_val = tmp2;
415         __ movptr(new_val, val);
416       }
417     }
418     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
419     if (needs_post_barrier) {
420       g1_write_barrier_post(masm /*masm*/,
421                             tmp1 /* store_adr */,
422                             new_val /* new_val */,
423                             rthread /* thread */,
424                             tmp3 /* tmp */,
425                             tmp2 /* tmp2 */);
426     }
427   }
428   NOT_LP64(imasm->restore_bcp());
429 }
430 
431 #ifdef COMPILER1
432 
433 #undef __
434 #define __ ce->masm()->
435 
436 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
437   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
438   // At this point we know that marking is in progress.
439   // If do_load() is true then we have to emit the
440   // load of the previous value; otherwise it has already
441   // been loaded into _pre_val.
442 
443   __ bind(*stub->entry());
444   assert(stub->pre_val()->is_register(), "Precondition.");
445 
446   Register pre_val_reg = stub->pre_val()->as_register();
447 
448   if (stub->do_load()) {
449     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
450   }
451 
452   __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
453   __ jcc(Assembler::equal, *stub->continuation());
454   ce->store_parameter(stub->pre_val()->as_register(), 0);
455   __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
456   __ jmp(*stub->continuation());
457 
458 }
459 
460 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
461   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
462   __ bind(*stub->entry());
463   assert(stub->addr()->is_register(), "Precondition.");
464   assert(stub->new_val()->is_register(), "Precondition.");
465   Register new_val_reg = stub->new_val()->as_register();
466   __ cmpptr(new_val_reg, (int32_t) NULL_WORD);
467   __ jcc(Assembler::equal, *stub->continuation());
468   ce->store_parameter(stub->addr()->as_pointer_register(), 0);
469   __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
470   __ jmp(*stub->continuation());
471 }
472 
473 #undef __
474 
475 #define __ sasm->
476 
477 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
478   // Generated code assumes that buffer index is pointer sized.
479   STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
480 
481   __ prologue("g1_pre_barrier", false);
482   // arg0 : previous value of memory
483 
484   __ push(rax);
485   __ push(rdx);
486 
487   const Register pre_val = rax;
488   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
489   const Register tmp = rdx;
490 
491   NOT_LP64(__ get_thread(thread);)
492 
493   Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
494   Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
495   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
496 
497   Label done;
498   Label runtime;
499 
500   // Is marking still active?
501   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
502     __ cmpl(queue_active, 0);
503   } else {
504     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
505     __ cmpb(queue_active, 0);
506   }
507   __ jcc(Assembler::equal, done);
508 
509   // Can we store original value in the thread's buffer?
510 
511   __ movptr(tmp, queue_index);
512   __ testptr(tmp, tmp);
513   __ jcc(Assembler::zero, runtime);
514   __ subptr(tmp, wordSize);
515   __ movptr(queue_index, tmp);
516   __ addptr(tmp, buffer);
517 
518   // prev_val (rax)
519   __ load_parameter(0, pre_val);
520   __ movptr(Address(tmp, 0), pre_val);
521   __ jmp(done);
522 
523   __ bind(runtime);
524 
525   __ save_live_registers_no_oop_map(true);
526 
527   // load the pre-value
528   __ load_parameter(0, rcx);
529   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread);
530 
531   __ restore_live_registers(true);
532 
533   __ bind(done);
534 
535   __ pop(rdx);
536   __ pop(rax);
537 
538   __ epilogue();
539 }
540 
541 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
542   __ prologue("g1_post_barrier", false);
543 
544   // arg0: store_address
545   Address store_addr(rbp, 2*BytesPerWord);
546 
547   CardTableBarrierSet* ct =
548     barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
549 
550   Label done;
551   Label enqueued;
552   Label runtime;
553 
554   // At this point we know new_value is non-NULL and the new_value crosses regions.
555   // Must check to see if card is already dirty
556 
557   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
558 
559   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
560   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
561 
562   __ push(rax);
563   __ push(rcx);
564 
565   const Register cardtable = rax;
566   const Register card_addr = rcx;
567 
568   __ load_parameter(0, card_addr);
569   __ shrptr(card_addr, CardTable::card_shift);
570   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
571   // a valid address and therefore is not properly handled by the relocation code.
572   __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
573   __ addptr(card_addr, cardtable);
574 
575   NOT_LP64(__ get_thread(thread);)
576 
577   __ cmpb(Address(card_addr, 0), (int)G1CardTable::g1_young_card_val());
578   __ jcc(Assembler::equal, done);
579 
580   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
581   __ cmpb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
582   __ jcc(Assembler::equal, done);
583 
584   // storing region crossing non-NULL, card is clean.
585   // dirty card and log.
586 
587   __ movb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
588 
589   const Register tmp = rdx;
590   __ push(rdx);
591 
592   __ movptr(tmp, queue_index);
593   __ testptr(tmp, tmp);
594   __ jcc(Assembler::zero, runtime);
595   __ subptr(tmp, wordSize);
596   __ movptr(queue_index, tmp);
597   __ addptr(tmp, buffer);
598   __ movptr(Address(tmp, 0), card_addr);
599   __ jmp(enqueued);
600 
601   __ bind(runtime);
602 
603   __ save_live_registers_no_oop_map(true);
604 
605   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
606 
607   __ restore_live_registers(true);
608 
609   __ bind(enqueued);
610   __ pop(rdx);
611 
612   __ bind(done);
613   __ pop(rcx);
614   __ pop(rax);
615 
616   __ epilogue();
617 }
618 
619 #undef __
620 
621 #endif // COMPILER1