1 /*
   2  * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "gc/g1/g1BarrierSet.hpp"
  28 #include "gc/g1/g1BarrierSetAssembler.hpp"
  29 #include "gc/g1/g1BarrierSetRuntime.hpp"
  30 #include "gc/g1/g1CardTable.hpp"
  31 #include "gc/g1/g1ThreadLocalData.hpp"
  32 #include "gc/g1/heapRegion.hpp"
  33 #include "interpreter/interp_masm.hpp"
  34 #include "runtime/sharedRuntime.hpp"
  35 #include "utilities/macros.hpp"
  36 #ifdef COMPILER1
  37 #include "c1/c1_LIRAssembler.hpp"
  38 #include "c1/c1_MacroAssembler.hpp"
  39 #include "gc/g1/c1/g1BarrierSetC1.hpp"
  40 #endif
  41 
  42 #define __ masm->
  43 
  44 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
  45                                                             Register addr, Register count) {
  46   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
  47   // With G1, don't generate the call if we statically know that the target in uninitialized
  48   if (!dest_uninitialized) {
  49     Register tmp = O5;
  50     assert_different_registers(addr, count, tmp);
  51     Label filtered;
  52     // Is marking active?
  53     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
  54       __ ld(G2, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), tmp);
  55     } else {
  56       guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
  57       __ ldsb(G2, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), tmp);
  58     }
  59     // Is marking active?
  60     __ cmp_and_br_short(tmp, G0, Assembler::equal, Assembler::pt, filtered);
  61 
  62     __ save_frame(0);
  63     // Save the necessary global regs... will be used after.
  64     if (addr->is_global()) {
  65       __ mov(addr, L0);
  66     }
  67     if (count->is_global()) {
  68       __ mov(count, L1);
  69     }
  70     __ mov(addr->after_save(), O0);
  71     // Get the count into O1
  72     address slowpath = UseCompressedOops ? CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry)
  73                                          : CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry);
  74     __ call(slowpath);
  75     __ delayed()->mov(count->after_save(), O1);
  76     if (addr->is_global()) {
  77       __ mov(L0, addr);
  78     }
  79     if (count->is_global()) {
  80       __ mov(L1, count);
  81     }
  82     __ restore();
  83 
  84     __ bind(filtered);
  85     DEBUG_ONLY(__ set(0xDEADC0DE, tmp);) // we have killed tmp
  86   }
  87 }
  88 
  89 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
  90                                                              Register addr, Register count, Register tmp) {
  91   // Get some new fresh output registers.
  92   __ save_frame(0);
  93   __ mov(addr->after_save(), O0);
  94   __ call(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry));
  95   __ delayed()->mov(count->after_save(), O1);
  96   __ restore();
  97 }
  98 
  99 #undef __
 100 
 101 static address satb_log_enqueue_with_frame = NULL;
 102 static u_char* satb_log_enqueue_with_frame_end = NULL;
 103 
 104 static address satb_log_enqueue_frameless = NULL;
 105 static u_char* satb_log_enqueue_frameless_end = NULL;
 106 
 107 static int EnqueueCodeSize = 128 DEBUG_ONLY( + 256); // Instructions?
 108 
 109 static void generate_satb_log_enqueue(bool with_frame) {
 110   BufferBlob* bb = BufferBlob::create("enqueue_with_frame", EnqueueCodeSize);
 111   CodeBuffer buf(bb);
 112   MacroAssembler masm(&buf);
 113 
 114 #define __ masm.
 115 
 116   address start = __ pc();
 117   Register pre_val;
 118 
 119   Label refill, restart;
 120   if (with_frame) {
 121     __ save_frame(0);
 122     pre_val = I0;  // Was O0 before the save.
 123   } else {
 124     pre_val = O0;
 125   }
 126 
 127   int satb_q_index_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
 128   int satb_q_buf_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
 129 
 130   assert(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t) &&
 131          in_bytes(SATBMarkQueue::byte_width_of_buf()) == sizeof(intptr_t),
 132          "check sizes in assembly below");
 133 
 134   __ bind(restart);
 135 
 136   // Load the index into the SATB buffer. SATBMarkQueue::_index is a size_t
 137   // so ld_ptr is appropriate.
 138   __ ld_ptr(G2_thread, satb_q_index_byte_offset, L0);
 139 
 140   // index == 0?
 141   __ cmp_and_brx_short(L0, G0, Assembler::equal, Assembler::pn, refill);
 142 
 143   __ ld_ptr(G2_thread, satb_q_buf_byte_offset, L1);
 144   __ sub(L0, oopSize, L0);
 145 
 146   __ st_ptr(pre_val, L1, L0);  // [_buf + index] := I0
 147   if (!with_frame) {
 148     // Use return-from-leaf
 149     __ retl();
 150     __ delayed()->st_ptr(L0, G2_thread, satb_q_index_byte_offset);
 151   } else {
 152     // Not delayed.
 153     __ st_ptr(L0, G2_thread, satb_q_index_byte_offset);
 154   }
 155   if (with_frame) {
 156     __ ret();
 157     __ delayed()->restore();
 158   }
 159   __ bind(refill);
 160 
 161   address handle_zero =
 162     CAST_FROM_FN_PTR(address,
 163                      &G1SATBMarkQueueSet::handle_zero_index_for_thread);
 164   // This should be rare enough that we can afford to save all the
 165   // scratch registers that the calling context might be using.
 166   __ mov(G1_scratch, L0);
 167   __ mov(G3_scratch, L1);
 168   __ mov(G4, L2);
 169   // We need the value of O0 above (for the write into the buffer), so we
 170   // save and restore it.
 171   __ mov(O0, L3);
 172   // Since the call will overwrite O7, we save and restore that, as well.
 173   __ mov(O7, L4);
 174   __ call_VM_leaf(L5, handle_zero, G2_thread);
 175   __ mov(L0, G1_scratch);
 176   __ mov(L1, G3_scratch);
 177   __ mov(L2, G4);
 178   __ mov(L3, O0);
 179   __ br(Assembler::always, /*annul*/false, Assembler::pt, restart);
 180   __ delayed()->mov(L4, O7);
 181 
 182   if (with_frame) {
 183     satb_log_enqueue_with_frame = start;
 184     satb_log_enqueue_with_frame_end = __ pc();
 185   } else {
 186     satb_log_enqueue_frameless = start;
 187     satb_log_enqueue_frameless_end = __ pc();
 188   }
 189 
 190 #undef __
 191 }
 192 
 193 #define __ masm->
 194 
 195 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
 196                                                  Register obj,
 197                                                  Register index,
 198                                                  int offset,
 199                                                  Register pre_val,
 200                                                  Register tmp,
 201                                                  bool preserve_o_regs) {
 202   Label filtered;
 203 
 204   if (obj == noreg) {
 205     // We are not loading the previous value so make
 206     // sure that we don't trash the value in pre_val
 207     // with the code below.
 208     assert_different_registers(pre_val, tmp);
 209   } else {
 210     // We will be loading the previous value
 211     // in this code so...
 212     assert(offset == 0 || index == noreg, "choose one");
 213     assert(pre_val == noreg, "check this code");
 214   }
 215 
 216   // Is marking active?
 217   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 218     __ ld(G2, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), tmp);
 219   } else {
 220     guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 221     __ ldsb(G2, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), tmp);
 222   }
 223 
 224   // Is marking active?
 225   __ cmp_and_br_short(tmp, G0, Assembler::equal, Assembler::pt, filtered);
 226 
 227   // Do we need to load the previous value?
 228   if (obj != noreg) {
 229     // Load the previous value...
 230     if (index == noreg) {
 231       if (Assembler::is_simm13(offset)) {
 232         __ load_heap_oop(obj, offset, tmp);
 233       } else {
 234         __ set(offset, tmp);
 235         __ load_heap_oop(obj, tmp, tmp);
 236       }
 237     } else {
 238       __ load_heap_oop(obj, index, tmp);
 239     }
 240     // Previous value has been loaded into tmp
 241     pre_val = tmp;
 242   }
 243 
 244   assert(pre_val != noreg, "must have a real register");
 245 
 246   // Is the previous value null?
 247   __ cmp_and_brx_short(pre_val, G0, Assembler::equal, Assembler::pt, filtered);
 248 
 249   // OK, it's not filtered, so we'll need to call enqueue.  In the normal
 250   // case, pre_val will be a scratch G-reg, but there are some cases in
 251   // which it's an O-reg.  In the first case, do a normal call.  In the
 252   // latter, do a save here and call the frameless version.
 253 
 254   guarantee(pre_val->is_global() || pre_val->is_out(),
 255             "Or we need to think harder.");
 256 
 257   if (pre_val->is_global() && !preserve_o_regs) {
 258     __ call(satb_log_enqueue_with_frame);
 259     __ delayed()->mov(pre_val, O0);
 260   } else {
 261     __ save_frame(0);
 262     __ call(satb_log_enqueue_frameless);
 263     __ delayed()->mov(pre_val->after_save(), O0);
 264     __ restore();
 265   }
 266 
 267   __ bind(filtered);
 268 }
 269 
 270 #undef __
 271 
 272 static address dirty_card_log_enqueue = 0;
 273 static u_char* dirty_card_log_enqueue_end = 0;
 274 
 275 // This gets to assume that o0 contains the object address.
 276 static void generate_dirty_card_log_enqueue(jbyte* byte_map_base) {
 277   BufferBlob* bb = BufferBlob::create("dirty_card_enqueue", EnqueueCodeSize*2);
 278   CodeBuffer buf(bb);
 279   MacroAssembler masm(&buf);
 280 #define __ masm.
 281   address start = __ pc();
 282 
 283   Label not_already_dirty, restart, refill, young_card;
 284 
 285   __ srlx(O0, CardTable::card_shift, O0);
 286   AddressLiteral addrlit(byte_map_base);
 287   __ set(addrlit, O1); // O1 := <card table base>
 288   __ ldub(O0, O1, O2); // O2 := [O0 + O1]
 289 
 290   __ cmp_and_br_short(O2, G1CardTable::g1_young_card_val(), Assembler::equal, Assembler::pt, young_card);
 291 
 292   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
 293   __ ldub(O0, O1, O2); // O2 := [O0 + O1]
 294 
 295   assert(G1CardTable::dirty_card_val() == 0, "otherwise check this code");
 296   __ cmp_and_br_short(O2, G0, Assembler::notEqual, Assembler::pt, not_already_dirty);
 297 
 298   __ bind(young_card);
 299   // We didn't take the branch, so we're already dirty: return.
 300   // Use return-from-leaf
 301   __ retl();
 302   __ delayed()->nop();
 303 
 304   // Not dirty.
 305   __ bind(not_already_dirty);
 306 
 307   // Get O0 + O1 into a reg by itself
 308   __ add(O0, O1, O3);
 309 
 310   // First, dirty it.
 311   __ stb(G0, O3, G0);  // [cardPtr] := 0  (i.e., dirty).
 312 
 313   int dirty_card_q_index_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
 314   int dirty_card_q_buf_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
 315   __ bind(restart);
 316 
 317   // Load the index into the update buffer. DirtyCardQueue::_index is
 318   // a size_t so ld_ptr is appropriate here.
 319   __ ld_ptr(G2_thread, dirty_card_q_index_byte_offset, L0);
 320 
 321   // index == 0?
 322   __ cmp_and_brx_short(L0, G0, Assembler::equal, Assembler::pn, refill);
 323 
 324   __ ld_ptr(G2_thread, dirty_card_q_buf_byte_offset, L1);
 325   __ sub(L0, oopSize, L0);
 326 
 327   __ st_ptr(O3, L1, L0);  // [_buf + index] := I0
 328   // Use return-from-leaf
 329   __ retl();
 330   __ delayed()->st_ptr(L0, G2_thread, dirty_card_q_index_byte_offset);
 331 
 332   __ bind(refill);
 333   address handle_zero =
 334     CAST_FROM_FN_PTR(address,
 335                      &DirtyCardQueueSet::handle_zero_index_for_thread);
 336   // This should be rare enough that we can afford to save all the
 337   // scratch registers that the calling context might be using.
 338   __ mov(G1_scratch, L3);
 339   __ mov(G3_scratch, L5);
 340   // We need the value of O3 above (for the write into the buffer), so we
 341   // save and restore it.
 342   __ mov(O3, L6);
 343   // Since the call will overwrite O7, we save and restore that, as well.
 344   __ mov(O7, L4);
 345 
 346   __ call_VM_leaf(L7_thread_cache, handle_zero, G2_thread);
 347   __ mov(L3, G1_scratch);
 348   __ mov(L5, G3_scratch);
 349   __ mov(L6, O3);
 350   __ br(Assembler::always, /*annul*/false, Assembler::pt, restart);
 351   __ delayed()->mov(L4, O7);
 352 
 353   dirty_card_log_enqueue = start;
 354   dirty_card_log_enqueue_end = __ pc();
 355   // XXX Should have a guarantee here about not going off the end!
 356   // Does it already do so?  Do an experiment...
 357 
 358 #undef __
 359 
 360 }
 361 
 362 #define __ masm->
 363 
 364 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Register store_addr, Register new_val, Register tmp) {
 365   Label filtered;
 366   MacroAssembler* post_filter_masm = masm;
 367 
 368   if (new_val == G0) return;
 369 
 370   G1BarrierSet* bs = barrier_set_cast<G1BarrierSet>(BarrierSet::barrier_set());
 371 
 372   __ xor3(store_addr, new_val, tmp);
 373   __ srlx(tmp, HeapRegion::LogOfHRGrainBytes, tmp);
 374 
 375   __ cmp_and_brx_short(tmp, G0, Assembler::equal, Assembler::pt, filtered);
 376 
 377   // If the "store_addr" register is an "in" or "local" register, move it to
 378   // a scratch reg so we can pass it as an argument.
 379   bool use_scr = !(store_addr->is_global() || store_addr->is_out());
 380   // Pick a scratch register different from "tmp".
 381   Register scr = (tmp == G1_scratch ? G3_scratch : G1_scratch);
 382   // Make sure we use up the delay slot!
 383   if (use_scr) {
 384     post_filter_masm->mov(store_addr, scr);
 385   } else {
 386     post_filter_masm->nop();
 387   }
 388   __ save_frame(0);
 389   __ call(dirty_card_log_enqueue);
 390   if (use_scr) {
 391     __ delayed()->mov(scr, O0);
 392   } else {
 393     __ delayed()->mov(store_addr->after_save(), O0);
 394   }
 395   __ restore();
 396 
 397   __ bind(filtered);
 398 }
 399 
 400 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 401                                          Register val, Address dst, Register tmp) {
 402   bool in_heap = (decorators & IN_HEAP) != 0;
 403   bool as_normal = (decorators & AS_NORMAL) != 0;
 404   assert((decorators & IS_DEST_UNINITIALIZED) == 0, "unsupported");
 405 
 406   bool needs_pre_barrier = as_normal;
 407   // No need for post barrier if storing NULL
 408   bool needs_post_barrier = val != G0 && in_heap;
 409 
 410   bool is_array = (decorators & IS_ARRAY) != 0;
 411   bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
 412   bool precise = is_array || on_anonymous;
 413 
 414   Register index = dst.has_index() ? dst.index() : noreg;
 415   int disp = dst.has_disp() ? dst.disp() : 0;
 416 
 417   if (needs_pre_barrier) {
 418     // Load and record the previous value.
 419     g1_write_barrier_pre(masm, dst.base(), index, disp,
 420                          noreg /* pre_val */,
 421                          tmp, true /*preserve_o_regs*/);
 422   }
 423 
 424   Register new_val = val;
 425   if (needs_post_barrier) {
 426     // G1 barrier needs uncompressed oop for region cross check.
 427     if (UseCompressedOops && val != G0) {
 428       new_val = tmp;
 429       __ mov(val, new_val);
 430     }
 431   }
 432 
 433   BarrierSetAssembler::store_at(masm, decorators, type, val, dst, tmp);
 434 
 435   if (needs_post_barrier) {
 436     Register base = dst.base();
 437     if (precise) {
 438       if (!dst.has_index()) {
 439         __ add(base, disp, base);
 440       } else {
 441         assert(!dst.has_disp(), "not supported yet");
 442         __ add(base, index, base);
 443       }
 444     }
 445     g1_write_barrier_post(masm, base, new_val, tmp);
 446   }
 447 }
 448 
 449 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 450                                     Address src, Register dst, Register tmp) {
 451   bool on_oop = type == T_OBJECT || type == T_ARRAY;
 452   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
 453   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
 454   bool on_reference = on_weak || on_phantom;
 455   // Load the value of the referent field.
 456   ModRefBarrierSetAssembler::load_at(masm, decorators, type, src, dst, tmp);
 457   if (on_oop && on_reference) {
 458     // Generate the G1 pre-barrier code to log the value of
 459     // the referent field in an SATB buffer. Note with
 460     // these parameters the pre-barrier does not generate
 461     // the load of the previous value
 462 
 463     Register pre_val = dst;
 464     bool saved = false;
 465     if (pre_val->is_in()) {
 466       // The g1_write_barrier_pre method assumes that the pre_val
 467       // is not in an input register.
 468       __ save_frame_and_mov(0, pre_val, O0);
 469       pre_val = O0;
 470       saved = true;
 471     }
 472 
 473     g1_write_barrier_pre(masm, noreg /* obj */, noreg /* index */, 0 /* offset */,
 474                          pre_val /* pre_val */,
 475                          tmp /* tmp */,
 476                          true /* preserve_o_regs */);
 477 
 478     if (saved) {
 479       __ restore();
 480     }
 481   }
 482 }
 483 
 484 void G1BarrierSetAssembler::barrier_stubs_init() {
 485   if (dirty_card_log_enqueue == 0) {
 486     G1BarrierSet* bs = barrier_set_cast<G1BarrierSet>(BarrierSet::barrier_set());
 487     CardTable *ct = bs->card_table();
 488     generate_dirty_card_log_enqueue(ct->byte_map_base());
 489     assert(dirty_card_log_enqueue != 0, "postcondition.");
 490   }
 491   if (satb_log_enqueue_with_frame == 0) {
 492     generate_satb_log_enqueue(true);
 493     assert(satb_log_enqueue_with_frame != 0, "postcondition.");
 494   }
 495   if (satb_log_enqueue_frameless == 0) {
 496     generate_satb_log_enqueue(false);
 497     assert(satb_log_enqueue_frameless != 0, "postcondition.");
 498   }
 499 }
 500 
 501 #ifdef COMPILER1
 502 
 503 #undef __
 504 #define __ ce->masm()->
 505 
 506 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
 507   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 508   // At this point we know that marking is in progress.
 509   // If do_load() is true then we have to emit the
 510   // load of the previous value; otherwise it has already
 511   // been loaded into _pre_val.
 512 
 513   __ bind(*stub->entry());
 514 
 515   assert(stub->pre_val()->is_register(), "Precondition.");
 516   Register pre_val_reg = stub->pre_val()->as_register();
 517 
 518   if (stub->do_load()) {
 519     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
 520   }
 521 
 522   if (__ is_in_wdisp16_range(*stub->continuation())) {
 523     __ br_null(pre_val_reg, /*annul*/false, Assembler::pt, *stub->continuation());
 524   } else {
 525     __ cmp(pre_val_reg, G0);
 526     __ brx(Assembler::equal, false, Assembler::pn, *stub->continuation());
 527   }
 528   __ delayed()->nop();
 529 
 530   __ call(bs->pre_barrier_c1_runtime_code_blob()->code_begin());
 531   __ delayed()->mov(pre_val_reg, G4);
 532   __ br(Assembler::always, false, Assembler::pt, *stub->continuation());
 533   __ delayed()->nop();
 534 }
 535 
 536 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
 537   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 538   __ bind(*stub->entry());
 539 
 540   assert(stub->addr()->is_register(), "Precondition.");
 541   assert(stub->new_val()->is_register(), "Precondition.");
 542   Register addr_reg = stub->addr()->as_pointer_register();
 543   Register new_val_reg = stub->new_val()->as_register();
 544 
 545   if (__ is_in_wdisp16_range(*stub->continuation())) {
 546     __ br_null(new_val_reg, /*annul*/false, Assembler::pt, *stub->continuation());
 547   } else {
 548     __ cmp(new_val_reg, G0);
 549     __ brx(Assembler::equal, false, Assembler::pn, *stub->continuation());
 550   }
 551   __ delayed()->nop();
 552 
 553   __ call(bs->post_barrier_c1_runtime_code_blob()->code_begin());
 554   __ delayed()->mov(addr_reg, G4);
 555   __ br(Assembler::always, false, Assembler::pt, *stub->continuation());
 556   __ delayed()->nop();
 557 }
 558 
 559 #undef __
 560 #define __ sasm->
 561 
 562 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
 563   __ prologue("g1_pre_barrier", false);
 564 
 565   // G4: previous value of memory
 566 
 567   Register pre_val = G4;
 568   Register tmp  = G1_scratch;
 569   Register tmp2 = G3_scratch;
 570 
 571   Label refill, restart;
 572   int satb_q_active_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
 573   int satb_q_index_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
 574   int satb_q_buf_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
 575 
 576   // Is marking still active?
 577   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 578     __ ld(G2_thread, satb_q_active_byte_offset, tmp);
 579   } else {
 580     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 581     __ ldsb(G2_thread, satb_q_active_byte_offset, tmp);
 582   }
 583   __ cmp_and_br_short(tmp, G0, Assembler::notEqual, Assembler::pt, restart);
 584   __ retl();
 585   __ delayed()->nop();
 586 
 587   __ bind(restart);
 588   // Load the index into the SATB buffer. SATBMarkQueue::_index is a
 589   // size_t so ld_ptr is appropriate
 590   __ ld_ptr(G2_thread, satb_q_index_byte_offset, tmp);
 591 
 592   // index == 0?
 593   __ cmp_and_brx_short(tmp, G0, Assembler::equal, Assembler::pn, refill);
 594 
 595   __ ld_ptr(G2_thread, satb_q_buf_byte_offset, tmp2);
 596   __ sub(tmp, oopSize, tmp);
 597 
 598   __ st_ptr(pre_val, tmp2, tmp);  // [_buf + index] := <address_of_card>
 599   // Use return-from-leaf
 600   __ retl();
 601   __ delayed()->st_ptr(tmp, G2_thread, satb_q_index_byte_offset);
 602 
 603   __ bind(refill);
 604 
 605   __ save_live_registers_no_oop_map(true);
 606 
 607   __ call_VM_leaf(L7_thread_cache,
 608                   CAST_FROM_FN_PTR(address,
 609                                    G1SATBMarkQueueSet::handle_zero_index_for_thread),
 610                                    G2_thread);
 611 
 612   __ restore_live_registers(true);
 613 
 614   __ br(Assembler::always, /*annul*/false, Assembler::pt, restart);
 615   __ epilogue();
 616 }
 617 
 618 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
 619   __ prologue("g1_post_barrier", false);
 620 
 621   G1BarrierSet* bs = barrier_set_cast<G1BarrierSet>(BarrierSet::barrier_set());
 622 
 623   Register addr = G4;
 624   Register cardtable = G5;
 625   Register tmp  = G1_scratch;
 626   Register tmp2 = G3_scratch;
 627   jbyte* byte_map_base = bs->card_table()->byte_map_base();
 628 
 629   Label not_already_dirty, restart, refill, young_card;
 630 
 631 #ifdef _LP64
 632   __ srlx(addr, CardTable::card_shift, addr);
 633 #else
 634   __ srl(addr, CardTable::card_shift, addr);
 635 #endif
 636 
 637   AddressLiteral rs((address)byte_map_base);
 638   __ set(rs, cardtable);         // cardtable := <card table base>
 639   __ ldub(addr, cardtable, tmp); // tmp := [addr + cardtable]
 640 
 641   __ cmp_and_br_short(tmp, G1CardTable::g1_young_card_val(), Assembler::equal, Assembler::pt, young_card);
 642 
 643   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
 644   __ ldub(addr, cardtable, tmp); // tmp := [addr + cardtable]
 645 
 646   assert(G1CardTable::dirty_card_val() == 0, "otherwise check this code");
 647   __ cmp_and_br_short(tmp, G0, Assembler::notEqual, Assembler::pt, not_already_dirty);
 648 
 649   __ bind(young_card);
 650   // We didn't take the branch, so we're already dirty: return.
 651   // Use return-from-leaf
 652   __ retl();
 653   __ delayed()->nop();
 654 
 655   // Not dirty.
 656   __ bind(not_already_dirty);
 657 
 658   // Get cardtable + tmp into a reg by itself
 659   __ add(addr, cardtable, tmp2);
 660 
 661   // First, dirty it.
 662   __ stb(G0, tmp2, 0);  // [cardPtr] := 0  (i.e., dirty).
 663 
 664   Register tmp3 = cardtable;
 665   Register tmp4 = tmp;
 666 
 667   // these registers are now dead
 668   addr = cardtable = tmp = noreg;
 669 
 670   int dirty_card_q_index_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
 671   int dirty_card_q_buf_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
 672 
 673   __ bind(restart);
 674 
 675   // Get the index into the update buffer. DirtyCardQueue::_index is
 676   // a size_t so ld_ptr is appropriate here.
 677   __ ld_ptr(G2_thread, dirty_card_q_index_byte_offset, tmp3);
 678 
 679   // index == 0?
 680   __ cmp_and_brx_short(tmp3, G0, Assembler::equal,  Assembler::pn, refill);
 681 
 682   __ ld_ptr(G2_thread, dirty_card_q_buf_byte_offset, tmp4);
 683   __ sub(tmp3, oopSize, tmp3);
 684 
 685   __ st_ptr(tmp2, tmp4, tmp3);  // [_buf + index] := <address_of_card>
 686   // Use return-from-leaf
 687   __ retl();
 688   __ delayed()->st_ptr(tmp3, G2_thread, dirty_card_q_index_byte_offset);
 689 
 690   __ bind(refill);
 691 
 692   __ save_live_registers_no_oop_map(true);
 693 
 694   __ call_VM_leaf(L7_thread_cache,
 695                   CAST_FROM_FN_PTR(address,
 696                                    DirtyCardQueueSet::handle_zero_index_for_thread),
 697                                    G2_thread);
 698 
 699   __ restore_live_registers(true);
 700 
 701   __ br(Assembler::always, /*annul*/false, Assembler::pt, restart);
 702   __ epilogue();
 703 }
 704 
 705 #undef __
 706 
 707 #endif // COMPILER1