1 /*
   2  * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "gc/g1/g1BarrierSet.hpp"
  28 #include "gc/g1/g1BarrierSetAssembler.hpp"
  29 #include "gc/g1/g1BarrierSetRuntime.hpp"
  30 #include "gc/g1/g1CardTable.hpp"
  31 #include "gc/g1/g1ThreadLocalData.hpp"
  32 #include "gc/g1/heapRegion.hpp"
  33 #include "interpreter/interp_masm.hpp"
  34 #include "runtime/sharedRuntime.hpp"
  35 #include "utilities/macros.hpp"
  36 #ifdef COMPILER1
  37 #include "c1/c1_LIRAssembler.hpp"
  38 #include "c1/c1_MacroAssembler.hpp"
  39 #include "gc/g1/c1/g1BarrierSetC1.hpp"
  40 #endif
  41 
  42 #define __ masm->
  43 
  44 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
  45                                                             Register addr, Register count) {
  46   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
  47 
  48   if (!dest_uninitialized) {
  49     Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
  50 #ifndef _LP64
  51     __ push(thread);
  52     __ get_thread(thread);
  53 #endif
  54 
  55     Label filtered;
  56     Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
  57     // Is marking active?
  58     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
  59       __ cmpl(in_progress, 0);
  60     } else {
  61       assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
  62       __ cmpb(in_progress, 0);
  63     }
  64 
  65     NOT_LP64(__ pop(thread);)
  66 
  67     __ jcc(Assembler::equal, filtered);
  68 
  69     __ pusha();                      // push registers
  70 #ifdef _LP64
  71     if (count == c_rarg0) {
  72       if (addr == c_rarg1) {
  73         // exactly backwards!!
  74         __ xchgptr(c_rarg1, c_rarg0);
  75       } else {
  76         __ movptr(c_rarg1, count);
  77         __ movptr(c_rarg0, addr);
  78       }
  79     } else {
  80       __ movptr(c_rarg0, addr);
  81       __ movptr(c_rarg1, count);
  82     }
  83     if (UseCompressedOops) {
  84       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2);
  85     } else {
  86       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2);
  87     }
  88 #else
  89     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry),
  90                     addr, count);
  91 #endif
  92     __ popa();
  93 
  94     __ bind(filtered);
  95   }
  96 }
  97 
  98 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
  99                                                              Register addr, Register count, Register tmp) {
 100   __ pusha();             // push registers (overkill)
 101 #ifdef _LP64
 102   if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
 103     assert_different_registers(c_rarg1, addr);
 104     __ mov(c_rarg1, count);
 105     __ mov(c_rarg0, addr);
 106   } else {
 107     assert_different_registers(c_rarg0, count);
 108     __ mov(c_rarg0, addr);
 109     __ mov(c_rarg1, count);
 110   }
 111   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
 112 #else
 113   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry),
 114                   addr, count);
 115 #endif
 116   __ popa();
 117 }
 118 
 119 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 120                                     Register dst, Address src, Register tmp1, Register tmp_thread) {
 121   bool on_oop = type == T_OBJECT || type == T_ARRAY;
 122   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
 123   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
 124   bool on_reference = on_weak || on_phantom;
 125   ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 126   if (on_oop && on_reference) {
 127     const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
 128     NOT_LP64(__ get_thread(thread));
 129 
 130     // Generate the G1 pre-barrier code to log the value of
 131     // the referent field in an SATB buffer.
 132     g1_write_barrier_pre(masm /* masm */,
 133                          noreg /* obj */,
 134                          dst /* pre_val */,
 135                          thread /* thread */,
 136                          tmp1 /* tmp */,
 137                          true /* tosca_live */,
 138                          true /* expand_call */);
 139   }
 140 }
 141 
 142 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
 143                                                  Register obj,
 144                                                  Register pre_val,
 145                                                  Register thread,
 146                                                  Register tmp,
 147                                                  bool tosca_live,
 148                                                  bool expand_call) {
 149   // If expand_call is true then we expand the call_VM_leaf macro
 150   // directly to skip generating the check by
 151   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
 152 
 153 #ifdef _LP64
 154   assert(thread == r15_thread, "must be");
 155 #endif // _LP64
 156 
 157   Label done;
 158   Label runtime;
 159 
 160   assert(pre_val != noreg, "check this code");
 161 
 162   if (obj != noreg) {
 163     assert_different_registers(obj, pre_val, tmp);
 164     assert(pre_val != rax, "check this code");
 165   }
 166 
 167   Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
 168   Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
 169   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
 170 
 171   // Is marking active?
 172   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 173     __ cmpl(in_progress, 0);
 174   } else {
 175     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 176     __ cmpb(in_progress, 0);
 177   }
 178   __ jcc(Assembler::equal, done);
 179 
 180   // Do we need to load the previous value?
 181   if (obj != noreg) {
 182     __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
 183   }
 184 
 185   // Is the previous value null?
 186   __ cmpptr(pre_val, (int32_t) NULL_WORD);
 187   __ jcc(Assembler::equal, done);
 188 
 189   // Can we store original value in the thread's buffer?
 190   // Is index == 0?
 191   // (The index field is typed as size_t.)
 192 
 193   __ movptr(tmp, index);                   // tmp := *index_adr
 194   __ cmpptr(tmp, 0);                       // tmp == 0?
 195   __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
 196 
 197   __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
 198   __ movptr(index, tmp);                   // *index_adr := tmp
 199   __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
 200 
 201   // Record the previous value
 202   __ movptr(Address(tmp, 0), pre_val);
 203   __ jmp(done);
 204 
 205   __ bind(runtime);
 206   // save the live input values
 207   if(tosca_live) __ push(rax);
 208 
 209   if (obj != noreg && obj != rax)
 210     __ push(obj);
 211 
 212   if (pre_val != rax)
 213     __ push(pre_val);
 214 
 215   // Calling the runtime using the regular call_VM_leaf mechanism generates
 216   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
 217   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
 218   //
 219   // If we care generating the pre-barrier without a frame (e.g. in the
 220   // intrinsified Reference.get() routine) then ebp might be pointing to
 221   // the caller frame and so this check will most likely fail at runtime.
 222   //
 223   // Expanding the call directly bypasses the generation of the check.
 224   // So when we do not have have a full interpreter frame on the stack
 225   // expand_call should be passed true.
 226 
 227   NOT_LP64( __ push(thread); )
 228 
 229   if (expand_call) {
 230     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
 231 #ifdef _LP64
 232     if (c_rarg1 != thread) {
 233       __ mov(c_rarg1, thread);
 234     }
 235     if (c_rarg0 != pre_val) {
 236       __ mov(c_rarg0, pre_val);
 237     }
 238 #else
 239     __ push(thread);
 240     __ push(pre_val);
 241 #endif
 242     __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
 243   } else {
 244     __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
 245   }
 246 
 247   NOT_LP64( __ pop(thread); )
 248 
 249   // save the live input values
 250   if (pre_val != rax)
 251     __ pop(pre_val);
 252 
 253   if (obj != noreg && obj != rax)
 254     __ pop(obj);
 255 
 256   if(tosca_live) __ pop(rax);
 257 
 258   __ bind(done);
 259 }
 260 
 261 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
 262                                                   Register store_addr,
 263                                                   Register new_val,
 264                                                   Register thread,
 265                                                   Register tmp,
 266                                                   Register tmp2) {
 267 #ifdef _LP64
 268   assert(thread == r15_thread, "must be");
 269 #endif // _LP64
 270 
 271   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
 272   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
 273 
 274   CardTableBarrierSet* ct =
 275     barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
 276 
 277   Label done;
 278   Label runtime;
 279 
 280   // Does store cross heap regions?
 281 
 282   __ movptr(tmp, store_addr);
 283   __ xorptr(tmp, new_val);
 284   __ shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
 285   __ jcc(Assembler::equal, done);
 286 
 287   // crosses regions, storing NULL?
 288 
 289   __ cmpptr(new_val, (int32_t) NULL_WORD);
 290   __ jcc(Assembler::equal, done);
 291 
 292   // storing region crossing non-NULL, is card already dirty?
 293 
 294   const Register card_addr = tmp;
 295   const Register cardtable = tmp2;
 296 
 297   __ movptr(card_addr, store_addr);
 298   __ shrptr(card_addr, CardTable::card_shift);
 299   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
 300   // a valid address and therefore is not properly handled by the relocation code.
 301   __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
 302   __ addptr(card_addr, cardtable);
 303 
 304   __ cmpb(Address(card_addr, 0), (int)G1CardTable::g1_young_card_val());
 305   __ jcc(Assembler::equal, done);
 306 
 307   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
 308   __ cmpb(Address(card_addr, 0), (int)G1CardTable::dirty_card_val());
 309   __ jcc(Assembler::equal, done);
 310 
 311 
 312   // storing a region crossing, non-NULL oop, card is clean.
 313   // dirty card and log.
 314 
 315   __ movb(Address(card_addr, 0), (int)G1CardTable::dirty_card_val());
 316 
 317   __ cmpl(queue_index, 0);
 318   __ jcc(Assembler::equal, runtime);
 319   __ subl(queue_index, wordSize);
 320   __ movptr(tmp2, buffer);
 321 #ifdef _LP64
 322   __ movslq(rscratch1, queue_index);
 323   __ addq(tmp2, rscratch1);
 324   __ movq(Address(tmp2, 0), card_addr);
 325 #else
 326   __ addl(tmp2, queue_index);
 327   __ movl(Address(tmp2, 0), card_addr);
 328 #endif
 329   __ jmp(done);
 330 
 331   __ bind(runtime);
 332   // save the live input values
 333   __ push(store_addr);
 334   __ push(new_val);
 335 #ifdef _LP64
 336   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, r15_thread);
 337 #else
 338   __ push(thread);
 339   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
 340   __ pop(thread);
 341 #endif
 342   __ pop(new_val);
 343   __ pop(store_addr);
 344 
 345   __ bind(done);
 346 }
 347 
 348 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 349                                          Address dst, Register val, Register tmp1, Register tmp2) {
 350   bool in_heap = (decorators & IN_HEAP) != 0;
 351   bool as_normal = (decorators & AS_NORMAL) != 0;
 352   assert((decorators & IS_DEST_UNINITIALIZED) == 0, "unsupported");
 353 
 354   bool needs_pre_barrier = as_normal;
 355   bool needs_post_barrier = val != noreg && in_heap;
 356 
 357   Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
 358   Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
 359   // flatten object address if needed
 360   // We do it regardless of precise because we need the registers
 361   if (dst.index() == noreg && dst.disp() == 0) {
 362     if (dst.base() != tmp1) {
 363       __ movptr(tmp1, dst.base());
 364     }
 365   } else {
 366     __ lea(tmp1, dst);
 367   }
 368 
 369 #ifndef _LP64
 370   InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
 371 #endif
 372 
 373   NOT_LP64(__ get_thread(rcx));
 374   NOT_LP64(imasm->save_bcp());
 375 
 376   if (needs_pre_barrier) {
 377     g1_write_barrier_pre(masm /*masm*/,
 378                          tmp1 /* obj */,
 379                          tmp2 /* pre_val */,
 380                          rthread /* thread */,
 381                          tmp3  /* tmp */,
 382                          val != noreg /* tosca_live */,
 383                          false /* expand_call */);
 384   }
 385   if (val == noreg) {
 386     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
 387   } else {
 388     Register new_val = val;
 389     if (needs_post_barrier) {
 390       // G1 barrier needs uncompressed oop for region cross check.
 391       if (UseCompressedOops) {
 392         new_val = tmp2;
 393         __ movptr(new_val, val);
 394       }
 395     }
 396     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
 397     if (needs_post_barrier) {
 398       g1_write_barrier_post(masm /*masm*/,
 399                             tmp1 /* store_adr */,
 400                             new_val /* new_val */,
 401                             rthread /* thread */,
 402                             tmp3 /* tmp */,
 403                             tmp2 /* tmp2 */);
 404     }
 405   }
 406   NOT_LP64(imasm->restore_bcp());
 407 }
 408 
 409 #ifdef COMPILER1
 410 
 411 #undef __
 412 #define __ ce->masm()->
 413 
 414 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
 415   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 416   // At this point we know that marking is in progress.
 417   // If do_load() is true then we have to emit the
 418   // load of the previous value; otherwise it has already
 419   // been loaded into _pre_val.
 420 
 421   __ bind(*stub->entry());
 422   assert(stub->pre_val()->is_register(), "Precondition.");
 423 
 424   Register pre_val_reg = stub->pre_val()->as_register();
 425 
 426   if (stub->do_load()) {
 427     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
 428   }
 429 
 430   __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
 431   __ jcc(Assembler::equal, *stub->continuation());
 432   ce->store_parameter(stub->pre_val()->as_register(), 0);
 433   __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
 434   __ jmp(*stub->continuation());
 435 
 436 }
 437 
 438 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
 439   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 440   __ bind(*stub->entry());
 441   assert(stub->addr()->is_register(), "Precondition.");
 442   assert(stub->new_val()->is_register(), "Precondition.");
 443   Register new_val_reg = stub->new_val()->as_register();
 444   __ cmpptr(new_val_reg, (int32_t) NULL_WORD);
 445   __ jcc(Assembler::equal, *stub->continuation());
 446   ce->store_parameter(stub->addr()->as_pointer_register(), 0);
 447   __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
 448   __ jmp(*stub->continuation());
 449 }
 450 
 451 #undef __
 452 
 453 #define __ sasm->
 454 
 455 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
 456   __ prologue("g1_pre_barrier", false);
 457   // arg0 : previous value of memory
 458 
 459   __ push(rax);
 460   __ push(rdx);
 461 
 462   const Register pre_val = rax;
 463   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
 464   const Register tmp = rdx;
 465 
 466   NOT_LP64(__ get_thread(thread);)
 467 
 468   Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
 469   Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
 470   Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
 471 
 472   Label done;
 473   Label runtime;
 474 
 475   // Is marking still active?
 476   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 477     __ cmpl(queue_active, 0);
 478   } else {
 479     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 480     __ cmpb(queue_active, 0);
 481   }
 482   __ jcc(Assembler::equal, done);
 483 
 484   // Can we store original value in the thread's buffer?
 485 
 486   __ movptr(tmp, queue_index);
 487   __ testptr(tmp, tmp);
 488   __ jcc(Assembler::zero, runtime);
 489   __ subptr(tmp, wordSize);
 490   __ movptr(queue_index, tmp);
 491   __ addptr(tmp, buffer);
 492 
 493   // prev_val (rax)
 494   __ load_parameter(0, pre_val);
 495   __ movptr(Address(tmp, 0), pre_val);
 496   __ jmp(done);
 497 
 498   __ bind(runtime);
 499 
 500   __ save_live_registers_no_oop_map(true);
 501 
 502   // load the pre-value
 503   __ load_parameter(0, rcx);
 504   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), rcx, thread);
 505 
 506   __ restore_live_registers(true);
 507 
 508   __ bind(done);
 509 
 510   __ pop(rdx);
 511   __ pop(rax);
 512 
 513   __ epilogue();
 514 }
 515 
 516 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
 517   __ prologue("g1_post_barrier", false);
 518 
 519   // arg0: store_address
 520   Address store_addr(rbp, 2*BytesPerWord);
 521 
 522   CardTableBarrierSet* ct =
 523     barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
 524 
 525   Label done;
 526   Label enqueued;
 527   Label runtime;
 528 
 529   // At this point we know new_value is non-NULL and the new_value crosses regions.
 530   // Must check to see if card is already dirty
 531 
 532   const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
 533 
 534   Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
 535   Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
 536 
 537   __ push(rax);
 538   __ push(rcx);
 539 
 540   const Register cardtable = rax;
 541   const Register card_addr = rcx;
 542 
 543   __ load_parameter(0, card_addr);
 544   __ shrptr(card_addr, CardTable::card_shift);
 545   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
 546   // a valid address and therefore is not properly handled by the relocation code.
 547   __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
 548   __ addptr(card_addr, cardtable);
 549 
 550   NOT_LP64(__ get_thread(thread);)
 551 
 552   __ cmpb(Address(card_addr, 0), (int)G1CardTable::g1_young_card_val());
 553   __ jcc(Assembler::equal, done);
 554 
 555   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
 556   __ cmpb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
 557   __ jcc(Assembler::equal, done);
 558 
 559   // storing region crossing non-NULL, card is clean.
 560   // dirty card and log.
 561 
 562   __ movb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
 563 
 564   const Register tmp = rdx;
 565   __ push(rdx);
 566 
 567   __ movptr(tmp, queue_index);
 568   __ testptr(tmp, tmp);
 569   __ jcc(Assembler::zero, runtime);
 570   __ subptr(tmp, wordSize);
 571   __ movptr(queue_index, tmp);
 572   __ addptr(tmp, buffer);
 573   __ movptr(Address(tmp, 0), card_addr);
 574   __ jmp(enqueued);
 575 
 576   __ bind(runtime);
 577 
 578   __ save_live_registers_no_oop_map(true);
 579 
 580   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
 581 
 582   __ restore_live_registers(true);
 583 
 584   __ bind(enqueued);
 585   __ pop(rdx);
 586 
 587   __ bind(done);
 588   __ pop(rcx);
 589   __ pop(rax);
 590 
 591   __ epilogue();
 592 }
 593 
 594 #undef __
 595 
 596 #endif // COMPILER1