1 /*
   2  * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2018, SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "gc/g1/g1BarrierSet.hpp"
  29 #include "gc/g1/g1BarrierSetAssembler.hpp"
  30 #include "gc/g1/g1BarrierSetRuntime.hpp"
  31 #include "gc/g1/g1CardTable.hpp"
  32 #include "gc/g1/g1ThreadLocalData.hpp"
  33 #include "gc/g1/heapRegion.hpp"
  34 #include "interpreter/interp_masm.hpp"
  35 #include "runtime/sharedRuntime.hpp"
  36 #ifdef COMPILER1
  37 #include "c1/c1_LIRAssembler.hpp"
  38 #include "c1/c1_MacroAssembler.hpp"
  39 #include "gc/g1/c1/g1BarrierSetC1.hpp"
  40 #endif
  41 
  42 #define __ masm->
  43 
  44 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
  45                                                             Register from, Register to, Register count,
  46                                                             Register preserve1, Register preserve2) {
  47   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
  48   // With G1, don't generate the call if we statically know that the target in uninitialized
  49   if (!dest_uninitialized) {
  50     int spill_slots = 3;
  51     if (preserve1 != noreg) { spill_slots++; }
  52     if (preserve2 != noreg) { spill_slots++; }
  53     const int frame_size = align_up(frame::abi_reg_args_size + spill_slots * BytesPerWord, frame::alignment_in_bytes);
  54     Label filtered;
  55 
  56     // Is marking active?
  57     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
  58       __ lwz(R0, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
  59     } else {
  60       guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
  61       __ lbz(R0, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
  62     }
  63     __ cmpdi(CCR0, R0, 0);
  64     __ beq(CCR0, filtered);
  65 
  66     __ save_LR_CR(R0);
  67     __ push_frame(frame_size, R0);
  68     int slot_nr = 0;
  69     __ std(from,  frame_size - (++slot_nr) * wordSize, R1_SP);
  70     __ std(to,    frame_size - (++slot_nr) * wordSize, R1_SP);
  71     __ std(count, frame_size - (++slot_nr) * wordSize, R1_SP);
  72     if (preserve1 != noreg) { __ std(preserve1, frame_size - (++slot_nr) * wordSize, R1_SP); }
  73     if (preserve2 != noreg) { __ std(preserve2, frame_size - (++slot_nr) * wordSize, R1_SP); }
  74 
  75     if (UseCompressedOops) {
  76       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), to, count);
  77     } else {
  78       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), to, count);
  79     }
  80 
  81     slot_nr = 0;
  82     __ ld(from,  frame_size - (++slot_nr) * wordSize, R1_SP);
  83     __ ld(to,    frame_size - (++slot_nr) * wordSize, R1_SP);
  84     __ ld(count, frame_size - (++slot_nr) * wordSize, R1_SP);
  85     if (preserve1 != noreg) { __ ld(preserve1, frame_size - (++slot_nr) * wordSize, R1_SP); }
  86     if (preserve2 != noreg) { __ ld(preserve2, frame_size - (++slot_nr) * wordSize, R1_SP); }
  87     __ addi(R1_SP, R1_SP, frame_size); // pop_frame()
  88     __ restore_LR_CR(R0);
  89 
  90     __ bind(filtered);
  91   }
  92 }
  93 
  94 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
  95                                                              Register addr, Register count, Register preserve) {
  96   int spill_slots = (preserve != noreg) ? 1 : 0;
  97   const int frame_size = align_up(frame::abi_reg_args_size + spill_slots * BytesPerWord, frame::alignment_in_bytes);
  98 
  99   __ save_LR_CR(R0);
 100   __ push_frame(frame_size, R0);
 101   if (preserve != noreg) { __ std(preserve, frame_size - 1 * wordSize, R1_SP); }
 102   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), addr, count);
 103   if (preserve != noreg) { __ ld(preserve, frame_size - 1 * wordSize, R1_SP); }
 104   __ addi(R1_SP, R1_SP, frame_size); // pop_frame();
 105   __ restore_LR_CR(R0);
 106 }
 107 
 108 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, DecoratorSet decorators, Register obj, RegisterOrConstant ind_or_offs, Register pre_val,
 109                                                  Register tmp1, Register tmp2, bool needs_frame) {
 110   bool not_null  = (decorators & IS_NOT_NULL) != 0,
 111        preloaded = obj == noreg;
 112   Register nv_save = noreg;
 113 
 114   if (preloaded) {
 115     // We are not loading the previous value so make
 116     // sure that we don't trash the value in pre_val
 117     // with the code below.
 118     assert_different_registers(pre_val, tmp1, tmp2);
 119     if (pre_val->is_volatile()) {
 120       nv_save = !tmp1->is_volatile() ? tmp1 : tmp2;
 121       assert(!nv_save->is_volatile(), "need one nv temp register if pre_val lives in volatile register");
 122     }
 123   }
 124 
 125   Label runtime, filtered;
 126 
 127   // Is marking active?
 128   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 129     __ lwz(tmp1, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
 130   } else {
 131     guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 132     __ lbz(tmp1, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
 133   }
 134   __ cmpdi(CCR0, tmp1, 0);
 135   __ beq(CCR0, filtered);
 136 
 137   // Do we need to load the previous value?
 138   if (!preloaded) {
 139     // Load the previous value...
 140     if (UseCompressedOops) {
 141       __ lwz(pre_val, ind_or_offs, obj);
 142     } else {
 143       __ ld(pre_val, ind_or_offs, obj);
 144     }
 145     // Previous value has been loaded into Rpre_val.
 146   }
 147   assert(pre_val != noreg, "must have a real register");
 148 
 149   // Is the previous value null?
 150   if (preloaded && not_null) {
 151 #ifdef ASSERT
 152     __ cmpdi(CCR0, pre_val, 0);
 153     __ asm_assert_ne("null oop not allowed (G1 pre)", 0x321); // Checked by caller.
 154 #endif
 155   } else {
 156     __ cmpdi(CCR0, pre_val, 0);
 157     __ beq(CCR0, filtered);
 158   }
 159 
 160   if (!preloaded && UseCompressedOops) {
 161     __ decode_heap_oop_not_null(pre_val);
 162   }
 163 
 164   // OK, it's not filtered, so we'll need to call enqueue. In the normal
 165   // case, pre_val will be a scratch G-reg, but there are some cases in
 166   // which it's an O-reg. In the first case, do a normal call. In the
 167   // latter, do a save here and call the frameless version.
 168 
 169   // Can we store original value in the thread's buffer?
 170   // Is index == 0?
 171   // (The index field is typed as size_t.)
 172   const Register Rbuffer = tmp1, Rindex = tmp2;
 173 
 174   __ ld(Rindex, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 175   __ cmpdi(CCR0, Rindex, 0);
 176   __ beq(CCR0, runtime); // If index == 0, goto runtime.
 177   __ ld(Rbuffer, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
 178 
 179   __ addi(Rindex, Rindex, -wordSize); // Decrement index.
 180   __ std(Rindex, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 181 
 182   // Record the previous value.
 183   __ stdx(pre_val, Rbuffer, Rindex);
 184   __ b(filtered);
 185 
 186   __ bind(runtime);
 187 
 188   // May need to preserve LR. Also needed if current frame is not compatible with C calling convention.
 189   if (needs_frame) {
 190     __ save_LR_CR(tmp1);
 191     __ push_frame_reg_args(0, tmp2);
 192   }
 193 
 194   if (pre_val->is_volatile() && preloaded) { __ mr(nv_save, pre_val); } // Save pre_val across C call if it was preloaded.
 195   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, R16_thread);
 196   if (pre_val->is_volatile() && preloaded) { __ mr(pre_val, nv_save); } // restore
 197 
 198   if (needs_frame) {
 199     __ pop_frame();
 200     __ restore_LR_CR(tmp1);
 201   }
 202 
 203   __ bind(filtered);
 204 }
 205 
 206 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators, Register store_addr, Register new_val,
 207                                                   Register tmp1, Register tmp2, Register tmp3) {
 208   bool not_null = (decorators & IS_NOT_NULL) != 0;
 209 
 210   Label runtime, filtered;
 211   assert_different_registers(store_addr, new_val, tmp1, tmp2);
 212 
 213   CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
 214   assert(sizeof(*ct->card_table()->byte_map_base()) == sizeof(jbyte), "adjust this code");
 215 
 216   // Does store cross heap regions?
 217   __ xorr(tmp1, store_addr, new_val);
 218   __ srdi_(tmp1, tmp1, HeapRegion::LogOfHRGrainBytes);
 219   __ beq(CCR0, filtered);
 220 
 221   // Crosses regions, storing NULL?
 222   if (not_null) {
 223 #ifdef ASSERT
 224     __ cmpdi(CCR0, new_val, 0);
 225     __ asm_assert_ne("null oop not allowed (G1 post)", 0x322); // Checked by caller.
 226 #endif
 227   } else {
 228     __ cmpdi(CCR0, new_val, 0);
 229     __ beq(CCR0, filtered);
 230   }
 231 
 232   // Storing region crossing non-NULL, is card already dirty?
 233   const Register Rcard_addr = tmp1;
 234   Register Rbase = tmp2;
 235   __ load_const_optimized(Rbase, (address)(ct->card_table()->byte_map_base()), /*temp*/ tmp3);
 236 
 237   __ srdi(Rcard_addr, store_addr, CardTable::card_shift);
 238 
 239   // Get the address of the card.
 240   __ lbzx(/*card value*/ tmp3, Rbase, Rcard_addr);
 241   __ cmpwi(CCR0, tmp3, (int)G1CardTable::g1_young_card_val());
 242   __ beq(CCR0, filtered);
 243 
 244   __ membar(Assembler::StoreLoad);
 245   __ lbzx(/*card value*/ tmp3, Rbase, Rcard_addr);  // Reload after membar.
 246   __ cmpwi(CCR0, tmp3 /* card value */, (int)G1CardTable::dirty_card_val());
 247   __ beq(CCR0, filtered);
 248 
 249   // Storing a region crossing, non-NULL oop, card is clean.
 250   // Dirty card and log.
 251   __ li(tmp3, (int)G1CardTable::dirty_card_val());
 252   //release(); // G1: oops are allowed to get visible after dirty marking.
 253   __ stbx(tmp3, Rbase, Rcard_addr);
 254 
 255   __ add(Rcard_addr, Rbase, Rcard_addr); // This is the address which needs to get enqueued.
 256   Rbase = noreg; // end of lifetime
 257 
 258   const Register Rqueue_index = tmp2,
 259                  Rqueue_buf   = tmp3;
 260   __ ld(Rqueue_index, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()), R16_thread);
 261   __ cmpdi(CCR0, Rqueue_index, 0);
 262   __ beq(CCR0, runtime); // index == 0 then jump to runtime
 263   __ ld(Rqueue_buf, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()), R16_thread);
 264 
 265   __ addi(Rqueue_index, Rqueue_index, -wordSize); // decrement index
 266   __ std(Rqueue_index, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()), R16_thread);
 267 
 268   __ stdx(Rcard_addr, Rqueue_buf, Rqueue_index); // store card
 269   __ b(filtered);
 270 
 271   __ bind(runtime);
 272 
 273   // Save the live input values.
 274   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), Rcard_addr, R16_thread);
 275 
 276   __ bind(filtered);
 277 }
 278 
 279 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 280                                        Register base, RegisterOrConstant ind_or_offs, Register val,
 281                                        Register tmp1, Register tmp2, Register tmp3, bool needs_frame) {
 282   bool is_array = (decorators & IS_ARRAY) != 0;
 283   bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
 284   bool precise = is_array || on_anonymous;
 285   // Load and record the previous value.
 286   g1_write_barrier_pre(masm, decorators, base, ind_or_offs,
 287                        tmp1, tmp2, tmp3, needs_frame);
 288 
 289   BarrierSetAssembler::store_at(masm, decorators, type, base, ind_or_offs, val, tmp1, tmp2, tmp3, needs_frame);
 290 
 291   // No need for post barrier if storing NULL
 292   if (val != noreg) {
 293     if (precise) {
 294       if (ind_or_offs.is_constant()) {
 295         __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp1);
 296       } else {
 297         __ add(base, ind_or_offs.as_register(), base);
 298       }
 299     }
 300     g1_write_barrier_post(masm, decorators, base, val, tmp1, tmp2, tmp3);
 301   }
 302 }
 303 
 304 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 305                                     Register base, RegisterOrConstant ind_or_offs, Register dst,
 306                                     Register tmp1, Register tmp2, bool needs_frame, Label *L_handle_null) {
 307   bool on_oop = type == T_OBJECT || type == T_ARRAY;
 308   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
 309   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
 310   bool on_reference = on_weak || on_phantom;
 311   Label done;
 312   if (on_oop && on_reference && L_handle_null == NULL) { L_handle_null = &done; }
 313   // Load the value of the referent field.
 314   ModRefBarrierSetAssembler::load_at(masm, decorators, type, base, ind_or_offs, dst, tmp1, tmp2, needs_frame, L_handle_null);
 315   if (on_oop && on_reference) {
 316     // Generate the G1 pre-barrier code to log the value of
 317     // the referent field in an SATB buffer. Note with
 318     // these parameters the pre-barrier does not generate
 319     // the load of the previous value
 320     // We only reach here if value is not null.
 321     g1_write_barrier_pre(masm, decorators | IS_NOT_NULL, noreg /* obj */, (intptr_t)0, dst /* pre_val */,
 322                          tmp1, tmp2, needs_frame);
 323   }
 324   __ bind(done);
 325 }
 326 
 327 void G1BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value, Register tmp1, Register tmp2, bool needs_frame) {
 328   Label done, not_weak;
 329   __ cmpdi(CCR0, value, 0);
 330   __ beq(CCR0, done);         // Use NULL as-is.
 331 
 332   __ clrrdi(tmp1, value, JNIHandles::weak_tag_size);
 333   __ andi_(tmp2, value, JNIHandles::weak_tag_mask);
 334   __ ld(value, 0, tmp1);      // Resolve (untagged) jobject.
 335 
 336   __ beq(CCR0, not_weak);     // Test for jweak tag.
 337   __ verify_oop(value);
 338   g1_write_barrier_pre(masm, IN_NATIVE | ON_PHANTOM_OOP_REF,
 339                        noreg, noreg, value,
 340                        tmp1, tmp2, needs_frame);
 341   __ bind(not_weak);
 342   __ verify_oop(value);
 343   __ bind(done);
 344 }
 345 
 346 #ifdef COMPILER1
 347 
 348 #undef __
 349 #define __ ce->masm()->
 350 
 351 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
 352   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 353   // At this point we know that marking is in progress.
 354   // If do_load() is true then we have to emit the
 355   // load of the previous value; otherwise it has already
 356   // been loaded into _pre_val.
 357 
 358   __ bind(*stub->entry());
 359 
 360   assert(stub->pre_val()->is_register(), "Precondition.");
 361   Register pre_val_reg = stub->pre_val()->as_register();
 362 
 363   if (stub->do_load()) {
 364     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
 365   }
 366 
 367   __ cmpdi(CCR0, pre_val_reg, 0);
 368   __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CCR0, Assembler::equal), *stub->continuation());
 369 
 370   address c_code = bs->pre_barrier_c1_runtime_code_blob()->code_begin();
 371   //__ load_const_optimized(R0, c_code);
 372   __ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(c_code));
 373   __ std(pre_val_reg, -8, R1_SP); // Pass pre_val on stack.
 374   __ mtctr(R0);
 375   __ bctrl();
 376   __ b(*stub->continuation());
 377 }
 378 
 379 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
 380   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
 381   __ bind(*stub->entry());
 382 
 383   assert(stub->addr()->is_register(), "Precondition.");
 384   assert(stub->new_val()->is_register(), "Precondition.");
 385   Register addr_reg = stub->addr()->as_pointer_register();
 386   Register new_val_reg = stub->new_val()->as_register();
 387 
 388   __ cmpdi(CCR0, new_val_reg, 0);
 389   __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CCR0, Assembler::equal), *stub->continuation());
 390 
 391   address c_code = bs->post_barrier_c1_runtime_code_blob()->code_begin();
 392   //__ load_const_optimized(R0, c_code);
 393   __ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(c_code));
 394   __ mtctr(R0);
 395   __ mr(R0, addr_reg); // Pass addr in R0.
 396   __ bctrl();
 397   __ b(*stub->continuation());
 398 }
 399 
 400 #undef __
 401 #define __ sasm->
 402 
 403 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
 404   BarrierSet* bs = BarrierSet::barrier_set();
 405 
 406   __ set_info("g1_pre_barrier_slow_id", false);
 407 
 408   // Using stack slots: pre_val (pre-pushed), spill tmp, spill tmp2.
 409   const int stack_slots = 3;
 410   Register pre_val = R0; // previous value of memory
 411   Register tmp  = R14;
 412   Register tmp2 = R15;
 413 
 414   Label refill, restart, marking_not_active;
 415   int satb_q_active_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
 416   int satb_q_index_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
 417   int satb_q_buf_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
 418 
 419   // Spill
 420   __ std(tmp, -16, R1_SP);
 421   __ std(tmp2, -24, R1_SP);
 422 
 423   // Is marking still active?
 424   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 425     __ lwz(tmp, satb_q_active_byte_offset, R16_thread);
 426   } else {
 427     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 428     __ lbz(tmp, satb_q_active_byte_offset, R16_thread);
 429   }
 430   __ cmpdi(CCR0, tmp, 0);
 431   __ beq(CCR0, marking_not_active);
 432 
 433   __ bind(restart);
 434   // Load the index into the SATB buffer. SATBMarkQueue::_index is a
 435   // size_t so ld_ptr is appropriate.
 436   __ ld(tmp, satb_q_index_byte_offset, R16_thread);
 437 
 438   // index == 0?
 439   __ cmpdi(CCR0, tmp, 0);
 440   __ beq(CCR0, refill);
 441 
 442   __ ld(tmp2, satb_q_buf_byte_offset, R16_thread);
 443   __ ld(pre_val, -8, R1_SP); // Load from stack.
 444   __ addi(tmp, tmp, -oopSize);
 445 
 446   __ std(tmp, satb_q_index_byte_offset, R16_thread);
 447   __ stdx(pre_val, tmp2, tmp); // [_buf + index] := <address_of_card>
 448 
 449   __ bind(marking_not_active);
 450   // Restore temp registers and return-from-leaf.
 451   __ ld(tmp2, -24, R1_SP);
 452   __ ld(tmp, -16, R1_SP);
 453   __ blr();
 454 
 455   __ bind(refill);
 456   const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_slots) * BytesPerWord;
 457   __ save_volatile_gprs(R1_SP, -nbytes_save); // except R0
 458   __ mflr(R0);
 459   __ std(R0, _abi(lr), R1_SP);
 460   __ push_frame_reg_args(nbytes_save, R0); // dummy frame for C call
 461   __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1SATBMarkQueueSet::handle_zero_index_for_thread), R16_thread);
 462   __ pop_frame();
 463   __ ld(R0, _abi(lr), R1_SP);
 464   __ mtlr(R0);
 465   __ restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
 466   __ b(restart);
 467 }
 468 
 469 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
 470   G1BarrierSet* bs = barrier_set_cast<G1BarrierSet>(BarrierSet::barrier_set());
 471 
 472   __ set_info("g1_post_barrier_slow_id", false);
 473 
 474   // Using stack slots: spill addr, spill tmp2
 475   const int stack_slots = 2;
 476   Register tmp = R0;
 477   Register addr = R14;
 478   Register tmp2 = R15;
 479   jbyte* byte_map_base = bs->card_table()->byte_map_base();
 480 
 481   Label restart, refill, ret;
 482 
 483   // Spill
 484   __ std(addr, -8, R1_SP);
 485   __ std(tmp2, -16, R1_SP);
 486 
 487   __ srdi(addr, R0, CardTable::card_shift); // Addr is passed in R0.
 488   __ load_const_optimized(/*cardtable*/ tmp2, byte_map_base, tmp);
 489   __ add(addr, tmp2, addr);
 490   __ lbz(tmp, 0, addr); // tmp := [addr + cardtable]
 491 
 492   // Return if young card.
 493   __ cmpwi(CCR0, tmp, G1CardTable::g1_young_card_val());
 494   __ beq(CCR0, ret);
 495 
 496   // Return if sequential consistent value is already dirty.
 497   __ membar(Assembler::StoreLoad);
 498   __ lbz(tmp, 0, addr); // tmp := [addr + cardtable]
 499 
 500   __ cmpwi(CCR0, tmp, G1CardTable::dirty_card_val());
 501   __ beq(CCR0, ret);
 502 
 503   // Not dirty.
 504 
 505   // First, dirty it.
 506   __ li(tmp, G1CardTable::dirty_card_val());
 507   __ stb(tmp, 0, addr);
 508 
 509   int dirty_card_q_index_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
 510   int dirty_card_q_buf_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
 511 
 512   __ bind(restart);
 513 
 514   // Get the index into the update buffer. DirtyCardQueue::_index is
 515   // a size_t so ld_ptr is appropriate here.
 516   __ ld(tmp2, dirty_card_q_index_byte_offset, R16_thread);
 517 
 518   // index == 0?
 519   __ cmpdi(CCR0, tmp2, 0);
 520   __ beq(CCR0, refill);
 521 
 522   __ ld(tmp, dirty_card_q_buf_byte_offset, R16_thread);
 523   __ addi(tmp2, tmp2, -oopSize);
 524 
 525   __ std(tmp2, dirty_card_q_index_byte_offset, R16_thread);
 526   __ add(tmp2, tmp, tmp2);
 527   __ std(addr, 0, tmp2); // [_buf + index] := <address_of_card>
 528 
 529   // Restore temp registers and return-from-leaf.
 530   __ bind(ret);
 531   __ ld(tmp2, -16, R1_SP);
 532   __ ld(addr, -8, R1_SP);
 533   __ blr();
 534 
 535   __ bind(refill);
 536   const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_slots) * BytesPerWord;
 537   __ save_volatile_gprs(R1_SP, -nbytes_save); // except R0
 538   __ mflr(R0);
 539   __ std(R0, _abi(lr), R1_SP);
 540   __ push_frame_reg_args(nbytes_save, R0); // dummy frame for C call
 541   __ call_VM_leaf(CAST_FROM_FN_PTR(address, DirtyCardQueueSet::handle_zero_index_for_thread), R16_thread);
 542   __ pop_frame();
 543   __ ld(R0, _abi(lr), R1_SP);
 544   __ mtlr(R0);
 545   __ restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
 546   __ b(restart);
 547 }
 548 
 549 #undef __
 550 
 551 #endif // COMPILER1