1 /*
   2  * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
   3  * Copyright (c) 2012, 2021 SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "gc/shared/gcArguments.hpp"
  27 #include "gc/shared/gc_globals.hpp"
  28 #include "macroAssembler_ppc.hpp"
  29 #include "precompiled.hpp"
  30 #include "asm/macroAssembler.inline.hpp"
  31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  32 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  33 #include "gc/shenandoah/shenandoahForwarding.hpp"
  34 #include "gc/shenandoah/shenandoahHeap.hpp"
  35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  37 #include "gc/shenandoah/shenandoahRuntime.hpp"
  38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  39 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  40 #include "gc/shenandoah/mode/shenandoahMode.hpp"
  41 #include "interpreter/interpreter.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "runtime/thread.hpp"
  44 #include "utilities/globalDefinitions.hpp"
  45 #include "vm_version_ppc.hpp"
  46 
  47 #ifdef COMPILER1
  48 
  49 #include "c1/c1_LIRAssembler.hpp"
  50 #include "c1/c1_MacroAssembler.hpp"
  51 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  52 
  53 #endif
  54 
  55 #define __ masm->
  56 
  57 void ShenandoahBarrierSetAssembler::satb_write_barrier(MacroAssembler *masm,
  58                                                        Register base, RegisterOrConstant ind_or_offs,
  59                                                        Register tmp1, Register tmp2, Register tmp3,
  60                                                        MacroAssembler::PreservationLevel preservation_level) {
  61   if (ShenandoahSATBBarrier) {
  62     __ block_comment("satb_write_barrier (shenandoahgc) {");
  63     satb_write_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
  64     __ block_comment("} satb_write_barrier (shenandoahgc)");
  65   }
  66 }
  67 
  68 void ShenandoahBarrierSetAssembler::iu_barrier(MacroAssembler *masm,
  69                                                Register val,
  70                                                Register tmp1, Register tmp2,
  71                                                MacroAssembler::PreservationLevel preservation_level,
  72                                                DecoratorSet decorators) {
  73   // IU barriers are also employed to avoid resurrection of weak references,
  74   // even if Shenandoah does not operate in incremental update mode.
  75   if (ShenandoahIUBarrier || ShenandoahSATBBarrier) {
  76     __ block_comment("iu_barrier (shenandoahgc) {");
  77     satb_write_barrier_impl(masm, decorators, noreg, noreg, val, tmp1, tmp2, preservation_level);
  78     __ block_comment("} iu_barrier (shenandoahgc)");
  79   }
  80 }
  81 
  82 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
  83                                                            Register base, RegisterOrConstant ind_or_offs,
  84                                                            Register dst,
  85                                                            Register tmp1, Register tmp2,
  86                                                            MacroAssembler::PreservationLevel preservation_level) {
  87   if (ShenandoahLoadRefBarrier) {
  88     __ block_comment("load_reference_barrier (shenandoahgc) {");
  89     load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
  90     __ block_comment("} load_reference_barrier (shenandoahgc)");
  91   }
  92 }
  93 
  94 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
  95                                                        Register src, Register dst, Register count,
  96                                                        Register preserve1, Register preserve2) {
  97   Register R11_tmp = R11_scratch1;
  98 
  99   assert_different_registers(src, dst, count, R11_tmp, noreg);
 100   if (preserve1 != noreg) {
 101     // Technically not required, but likely to indicate an error.
 102     assert_different_registers(preserve1, preserve2);
 103   }
 104 
 105   /* ==== Check whether barrier is required (optimizations) ==== */
 106   // Fast path: Component type of array is not a reference type.
 107   if (!is_reference_type(type)) {
 108     return;
 109   }
 110 
 111   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 112 
 113   // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
 114   // any useful information.
 115   if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahIUBarrier && !ShenandoahLoadRefBarrier) {
 116     return;
 117   }
 118 
 119   __ block_comment("arraycopy_prologue (shenandoahgc) {");
 120   Label skip_prologue;
 121 
 122   // Fast path: Array is of length zero.
 123   __ cmpdi(CCR0, count, 0);
 124   __ beq(CCR0, skip_prologue);
 125 
 126   /* ==== Check whether barrier is required (gc state) ==== */
 127   __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
 128          R16_thread);
 129 
 130   // The set of garbage collection states requiring barriers depends on the available barrier types and the
 131   // type of the reference in question.
 132   // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
 133   // for the garbage collector.
 134   const int required_states = ShenandoahSATBBarrier && dest_uninitialized
 135                               ? ShenandoahHeap::HAS_FORWARDED
 136                               : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING;
 137 
 138   __ andi_(R11_tmp, R11_tmp, required_states);
 139   __ beq(CCR0, skip_prologue);
 140 
 141   /* ==== Invoke runtime ==== */
 142   // Save to-be-preserved registers.
 143   int highest_preserve_register_index = 0;
 144   {
 145     if (preserve1 != noreg && preserve1->is_volatile()) {
 146       __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 147     }
 148     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 149       __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 150     }
 151 
 152     __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 153     __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 154     __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 155 
 156     __ save_LR_CR(R11_tmp);
 157     __ push_frame_reg_args(-BytesPerWord * highest_preserve_register_index,
 158                            R11_tmp);
 159   }
 160 
 161   // Invoke runtime.
 162   address jrt_address = NULL;
 163   if (UseCompressedOops) {
 164     jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry);
 165   } else {
 166     jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry);
 167   }
 168   assert(jrt_address != nullptr, "jrt routine cannot be found");
 169 
 170   __ call_VM_leaf(jrt_address, src, dst, count);
 171 
 172   // Restore to-be-preserved registers.
 173   {
 174     __ pop_frame();
 175     __ restore_LR_CR(R11_tmp);
 176 
 177     __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 178     __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 179     __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 180 
 181     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 182       __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 183     }
 184     if (preserve1 != noreg && preserve1->is_volatile()) {
 185       __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 186     }
 187   }
 188 
 189   __ bind(skip_prologue);
 190   __ block_comment("} arraycopy_prologue (shenandoahgc)");
 191 }
 192 
 193 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 194                                                        Register dst, Register count,
 195                                                        Register preserve) {
 196   if (is_reference_type(type)) {
 197     __ block_comment("arraycopy_epilogue (shenandoahgc) {");
 198     gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
 199     __ block_comment("} arraycopy_epilogue (shenandoahgc)");
 200   }
 201 }
 202 
 203 // The to-be-enqueued value can either be determined
 204 // - dynamically by passing the reference's address information (load mode) or
 205 // - statically by passing a register the value is stored in (preloaded mode)
 206 //   - for performance optimizations in cases where the previous value is known (currently not implemented) and
 207 //   - for incremental-update barriers.
 208 //
 209 // decorators:  The previous value's decorator set.
 210 //              In "load mode", the value must equal '0'.
 211 // base:        Base register of the reference's address (load mode).
 212 //              In "preloaded mode", the register must equal 'noreg'.
 213 // ind_or_offs: Index or offset of the reference's address (load mode).
 214 //              If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
 215 // pre_val:     Register holding the to-be-stored value (preloaded mode).
 216 //              In "load mode", this register acts as a temporary register and must
 217 //              thus not be 'noreg'.  In "preloaded mode", its content will be sustained.
 218 // tmp1/tmp2:   Temporary registers, one of which must be non-volatile in "preloaded mode".
 219 void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
 220                                                             Register base, RegisterOrConstant ind_or_offs,
 221                                                             Register pre_val,
 222                                                             Register tmp1, Register tmp2,
 223                                                             MacroAssembler::PreservationLevel preservation_level) {
 224   assert_different_registers(tmp1, tmp2, pre_val, noreg);
 225 
 226   Label skip_barrier;
 227 
 228   /* ==== Determine necessary runtime invocation preservation measures ==== */
 229   const bool needs_frame           = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
 230   const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
 231   const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
 232 
 233   // Check whether marking is active.
 234   __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 235 
 236   __ andi_(tmp1, tmp1, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
 237   __ beq(CCR0, skip_barrier);
 238 
 239   /* ==== Determine the reference's previous value ==== */
 240   bool preloaded_mode = base == noreg;
 241   Register pre_val_save = noreg;
 242 
 243   if (preloaded_mode) {
 244     // Previous value has been passed to the method, so it must not be determined manually.
 245     // In case 'pre_val' is a volatile register, it must be saved across the C-call
 246     // as callers may depend on its value.
 247     // Unless the general purposes registers are saved anyway, one of the temporary registers
 248     // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
 249     if (!preserve_gp_registers && pre_val->is_volatile()) {
 250       pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
 251       assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
 252     }
 253 
 254     if ((decorators & IS_NOT_NULL) != 0) {
 255 #ifdef ASSERT
 256       __ cmpdi(CCR0, pre_val, 0);
 257       __ asm_assert_ne("null oop is not allowed");
 258 #endif // ASSERT
 259     } else {
 260       __ cmpdi(CCR0, pre_val, 0);
 261       __ beq(CCR0, skip_barrier);
 262     }
 263   } else {
 264     // Load from the reference address to determine the reference's current value (before the store is being performed).
 265     // Contrary to the given value in "preloaded mode", it is not necessary to preserve it.
 266     assert(decorators == 0, "decorator set must be empty");
 267     assert(base != noreg, "base must be a register");
 268     assert(!ind_or_offs.is_register() || ind_or_offs.as_register() != noreg, "ind_or_offs must be a register");
 269     if (UseCompressedOops) {
 270       __ lwz(pre_val, ind_or_offs, base);
 271     } else {
 272       __ ld(pre_val, ind_or_offs, base);
 273     }
 274 
 275     __ cmpdi(CCR0, pre_val, 0);
 276     __ beq(CCR0, skip_barrier);
 277 
 278     if (UseCompressedOops) {
 279       __ decode_heap_oop_not_null(pre_val);
 280     }
 281   }
 282 
 283   /* ==== Try to enqueue the to-be-stored value directly into thread's local SATB mark queue ==== */
 284   {
 285     Label runtime;
 286     Register Rbuffer = tmp1, Rindex = tmp2;
 287 
 288     // Check whether the queue has enough capacity to store another oop.
 289     // If not, jump to the runtime to commit the buffer and to allocate a new one.
 290     // (The buffer's index corresponds to the amount of remaining free space.)
 291     __ ld(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 292     __ cmpdi(CCR0, Rindex, 0);
 293     __ beq(CCR0, runtime); // If index == 0 (buffer is full), goto runtime.
 294 
 295     // Capacity suffices.  Decrement the queue's size by the size of one oop.
 296     // (The buffer is filled contrary to the heap's growing direction, i.e., it is filled downwards.)
 297     __ addi(Rindex, Rindex, -wordSize);
 298     __ std(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 299 
 300     // Enqueue the previous value and skip the invocation of the runtime.
 301     __ ld(Rbuffer, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
 302     __ stdx(pre_val, Rbuffer, Rindex);
 303     __ b(skip_barrier);
 304 
 305     __ bind(runtime);
 306   }
 307 
 308   /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
 309   // Save to-be-preserved registers.
 310   int nbytes_save = 0;
 311 
 312   if (needs_frame) {
 313     if (preserve_gp_registers) {
 314       nbytes_save = (preserve_fp_registers
 315                      ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
 316                      : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
 317       __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
 318     }
 319 
 320     __ save_LR_CR(tmp1);
 321     __ push_frame_reg_args(nbytes_save, tmp2);
 322   }
 323 
 324   if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
 325     assert(pre_val_save != noreg, "nv_save must not be noreg");
 326 
 327     // 'pre_val' register must be saved manually unless general-purpose are preserved in general.
 328     __ mr(pre_val_save, pre_val);
 329   }
 330 
 331   // Invoke runtime.
 332   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, R16_thread);
 333 
 334   // Restore to-be-preserved registers.
 335   if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
 336     __ mr(pre_val, pre_val_save);
 337   }
 338 
 339   if (needs_frame) {
 340     __ pop_frame();
 341     __ restore_LR_CR(tmp1);
 342 
 343     if (preserve_gp_registers) {
 344       __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
 345     }
 346   }
 347 
 348   __ bind(skip_barrier);
 349 }
 350 
 351 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler *masm, Register dst, Register tmp) {
 352   __ block_comment("resolve_forward_pointer_not_null (shenandoahgc) {");
 353 
 354   Register tmp1 = tmp,
 355            R0_tmp2 = R0;
 356   assert_different_registers(dst, tmp1, R0_tmp2, noreg);
 357 
 358   // If the object has been evacuated, the mark word layout is as follows:
 359   // | forwarding pointer (62-bit) | '11' (2-bit) |
 360 
 361   // The invariant that stack/thread pointers have the lowest two bits cleared permits retrieving
 362   // the forwarding pointer solely by inversing the lowest two bits.
 363   // This invariant follows inevitably from hotspot's minimal alignment.
 364   assert(markWord::marked_value <= (unsigned long) MinObjAlignmentInBytes,
 365          "marked value must not be higher than hotspot's minimal alignment");
 366 
 367   Label done;
 368 
 369   // Load the object's mark word.
 370   __ ld(tmp1, oopDesc::mark_offset_in_bytes(), dst);
 371 
 372   // Load the bit mask for the lock bits.
 373   __ li(R0_tmp2, markWord::lock_mask_in_place);
 374 
 375   // Check whether all bits matching the bit mask are set.
 376   // If that is the case, the object has been evacuated and the most significant bits form the forward pointer.
 377   __ andc_(R0_tmp2, R0_tmp2, tmp1);
 378 
 379   assert(markWord::lock_mask_in_place == markWord::marked_value,
 380          "marked value must equal the value obtained when all lock bits are being set");
 381   if (VM_Version::has_isel()) {
 382     __ xori(tmp1, tmp1, markWord::lock_mask_in_place);
 383     __ isel(dst, CCR0, Assembler::equal, false, tmp1);
 384   } else {
 385     __ bne(CCR0, done);
 386     __ xori(dst, tmp1, markWord::lock_mask_in_place);
 387   }
 388 
 389   __ bind(done);
 390   __ block_comment("} resolve_forward_pointer_not_null (shenandoahgc)");
 391 }
 392 
 393 // base:        Base register of the reference's address.
 394 // ind_or_offs: Index or offset of the reference's address (load mode).
 395 // dst:         Reference's address.  In case the object has been evacuated, this is the to-space version
 396 //              of that object.
 397 void ShenandoahBarrierSetAssembler::load_reference_barrier_impl(
 398     MacroAssembler *masm, DecoratorSet decorators,
 399     Register base, RegisterOrConstant ind_or_offs,
 400     Register dst,
 401     Register tmp1, Register tmp2,
 402     MacroAssembler::PreservationLevel preservation_level) {
 403   if (ind_or_offs.is_register()) {
 404     assert_different_registers(tmp1, tmp2, base, ind_or_offs.as_register(), dst, noreg);
 405   } else {
 406     assert_different_registers(tmp1, tmp2, base, dst, noreg);
 407   }
 408 
 409   Label skip_barrier;
 410 
 411   bool is_strong  = ShenandoahBarrierSet::is_strong_access(decorators);
 412   bool is_weak    = ShenandoahBarrierSet::is_weak_access(decorators);
 413   bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
 414   bool is_native  = ShenandoahBarrierSet::is_native_access(decorators);
 415   bool is_narrow  = UseCompressedOops && !is_native;
 416 
 417   /* ==== Check whether heap is stable ==== */
 418   __ lbz(tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 419 
 420   if (is_strong) {
 421     // For strong references, the heap is considered stable if "has forwarded" is not active.
 422     __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
 423     __ beq(CCR0, skip_barrier);
 424 #ifdef ASSERT
 425     // "evacuation" -> (implies) "has forwarded".  If we reach this code, "has forwarded" must thus be set.
 426     __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
 427     __ asm_assert_ne("'has forwarded' is missing");
 428 #endif // ASSERT
 429   } else {
 430     // For all non-strong references, the heap is considered stable if not any of "has forwarded",
 431     // "root set processing", and "weak reference processing" is active.
 432     // The additional phase conditions are in place to avoid the resurrection of weak references (see JDK-8266440).
 433     Label skip_fastpath;
 434     __ andi_(tmp1, tmp2, ShenandoahHeap::WEAK_ROOTS);
 435     __ bne(CCR0, skip_fastpath);
 436 
 437     __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
 438     __ beq(CCR0, skip_barrier);
 439 #ifdef ASSERT
 440     // "evacuation" -> (implies) "has forwarded".  If we reach this code, "has forwarded" must thus be set.
 441     __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
 442     __ asm_assert_ne("'has forwarded' is missing");
 443 #endif // ASSERT
 444 
 445     __ bind(skip_fastpath);
 446   }
 447 
 448   /* ==== Check whether region is in collection set ==== */
 449   if (is_strong) {
 450     // Shenandoah stores metadata on regions in a continuous area of memory in which a single byte corresponds to
 451     // an entire region of the shenandoah heap.  At present, only the least significant bit is of significance
 452     // and indicates whether the region is part of the collection set.
 453     //
 454     // All regions are of the same size and are always aligned by a power of two.
 455     // Any address can thus be shifted by a fixed number of bits to retrieve the address prefix shared by
 456     // all objects within that region (region identification bits).
 457     //
 458     //  | unused bits | region identification bits | object identification bits |
 459     //  (Region size depends on a couple of criteria, such as page size, user-provided arguments and the max heap size.
 460     //   The number of object identification bits can thus not be determined at compile time.)
 461     //
 462     // -------------------------------------------------------  <--- cs (collection set) base address
 463     // | lost space due to heap space base address                   -> 'ShenandoahHeap::in_cset_fast_test_addr()'
 464     // | (region identification bits contain heap base offset)
 465     // |------------------------------------------------------  <--- cs base address + (heap_base >> region size shift)
 466     // | collection set in the proper                                -> shift: 'region_size_bytes_shift_jint()'
 467     // |
 468     // |------------------------------------------------------  <--- cs base address + (heap_base >> region size shift)
 469     //                                                                               + number of regions
 470     __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
 471     __ srdi(tmp1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
 472     __ lbzx(tmp2, tmp1, tmp2);
 473     __ andi_(tmp2, tmp2, 1);
 474     __ beq(CCR0, skip_barrier);
 475   }
 476 
 477   /* ==== Invoke runtime ==== */
 478   // Save to-be-preserved registers.
 479   int nbytes_save = 0;
 480 
 481   const bool needs_frame           = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
 482   const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
 483   const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
 484 
 485   if (needs_frame) {
 486     if (preserve_gp_registers) {
 487       nbytes_save = (preserve_fp_registers
 488                      ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
 489                      : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
 490       __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
 491     }
 492 
 493     __ save_LR_CR(tmp1);
 494     __ push_frame_reg_args(nbytes_save, tmp1);
 495   }
 496 
 497   // Calculate the reference's absolute address.
 498   __ add(R4_ARG2, ind_or_offs, base);
 499 
 500   // Invoke runtime.
 501   address jrt_address = nullptr;
 502 
 503   if (is_strong) {
 504     if (is_narrow) {
 505       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
 506     } else {
 507       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
 508     }
 509   } else if (is_weak) {
 510     if (is_narrow) {
 511       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
 512     } else {
 513       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
 514     }
 515   } else {
 516     assert(is_phantom, "only remaining strength");
 517     assert(!is_narrow, "phantom access cannot be narrow");
 518     jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
 519   }
 520   assert(jrt_address != nullptr, "jrt routine cannot be found");
 521 
 522   __ call_VM_leaf(jrt_address, dst /* reference */, R4_ARG2 /* reference address */);
 523 
 524   // Restore to-be-preserved registers.
 525   if (preserve_gp_registers) {
 526     __ mr(R0, R3_RET);
 527   } else {
 528     __ mr_if_needed(dst, R3_RET);
 529   }
 530 
 531   if (needs_frame) {
 532     __ pop_frame();
 533     __ restore_LR_CR(tmp1);
 534 
 535     if (preserve_gp_registers) {
 536       __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
 537       __ mr(dst, R0);
 538     }
 539   }
 540 
 541   __ bind(skip_barrier);
 542 }
 543 
 544 // base:           Base register of the reference's address.
 545 // ind_or_offs:    Index or offset of the reference's address.
 546 // L_handle_null:  An optional label that will be jumped to if the reference is null.
 547 void ShenandoahBarrierSetAssembler::load_at(
 548     MacroAssembler *masm, DecoratorSet decorators, BasicType type,
 549     Register base, RegisterOrConstant ind_or_offs, Register dst,
 550     Register tmp1, Register tmp2,
 551     MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
 552   // Register must not clash, except 'base' and 'dst'.
 553   if (ind_or_offs.is_register()) {
 554     if (base != noreg) {
 555       assert_different_registers(tmp1, tmp2, base, ind_or_offs.register_or_noreg(), R0, noreg);
 556     }
 557     assert_different_registers(tmp1, tmp2, dst, ind_or_offs.register_or_noreg(), R0, noreg);
 558   } else {
 559     if (base == noreg) {
 560       assert_different_registers(tmp1, tmp2, base, R0, noreg);
 561     }
 562     assert_different_registers(tmp1, tmp2, dst, R0, noreg);
 563   }
 564 
 565   /* ==== Apply load barrier, if required ==== */
 566   if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
 567     assert(is_reference_type(type), "need_load_reference_barrier must check whether type is a reference type");
 568 
 569     // If 'dst' clashes with either 'base' or 'ind_or_offs', use an intermediate result register
 570     // to keep the values of those alive until the load reference barrier is applied.
 571     Register intermediate_dst = (dst == base || (ind_or_offs.is_register() && dst == ind_or_offs.as_register()))
 572                                 ? tmp2
 573                                 : dst;
 574 
 575     BarrierSetAssembler::load_at(masm, decorators, type,
 576                                  base, ind_or_offs,
 577                                  intermediate_dst,
 578                                  tmp1, noreg,
 579                                  preservation_level, L_handle_null);
 580 
 581     load_reference_barrier(masm, decorators,
 582                            base, ind_or_offs,
 583                            intermediate_dst,
 584                            tmp1, R0,
 585                            preservation_level);
 586 
 587     __ mr_if_needed(dst, intermediate_dst);
 588   } else {
 589     BarrierSetAssembler::load_at(masm, decorators, type,
 590                                  base, ind_or_offs,
 591                                  dst,
 592                                  tmp1, tmp2,
 593                                  preservation_level, L_handle_null);
 594   }
 595 
 596   /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
 597   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
 598     iu_barrier(masm, dst, tmp1, tmp2, preservation_level);
 599   }
 600 }
 601 
 602 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
 603   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 604       return;
 605   }
 606 
 607   ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
 608   CardTable* ct = ctbs->card_table();
 609   assert_different_registers(base, tmp, R0);
 610 
 611   if (ind_or_offs.is_constant()) {
 612     __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
 613   } else {
 614     __ add(base, ind_or_offs.as_register(), base);
 615   }
 616 
 617   __ load_const_optimized(tmp, (address)ct->byte_map_base(), R0);
 618   __ srdi(base, base, CardTable::card_shift());
 619   __ li(R0, CardTable::dirty_card_val());
 620   __ stbx(R0, tmp, base);
 621 }
 622 
 623 // base:        Base register of the reference's address.
 624 // ind_or_offs: Index or offset of the reference's address.
 625 // val:         To-be-stored value/reference's new value.
 626 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
 627                                              Register base, RegisterOrConstant ind_or_offs, Register val,
 628                                              Register tmp1, Register tmp2, Register tmp3,
 629                                              MacroAssembler::PreservationLevel preservation_level) {
 630   if (is_reference_type(type)) {
 631     if (ShenandoahSATBBarrier) {
 632       satb_write_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
 633     }
 634 
 635     if (ShenandoahIUBarrier && val != noreg) {
 636       iu_barrier(masm, val, tmp1, tmp2, preservation_level, decorators);
 637     }
 638   }
 639 
 640   BarrierSetAssembler::store_at(masm, decorators, type,
 641                                 base, ind_or_offs,
 642                                 val,
 643                                 tmp1, tmp2, tmp3,
 644                                 preservation_level);
 645 
 646   // No need for post barrier if storing NULL
 647   if (is_reference_type(type) && val != noreg) {
 648     store_check(masm, base, ind_or_offs, tmp1);
 649   }
 650 }
 651 
 652 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
 653                                                                   Register dst, Register jni_env, Register obj,
 654                                                                   Register tmp, Label &slowpath) {
 655   __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
 656 
 657   assert_different_registers(jni_env, obj, tmp);
 658 
 659   Label done;
 660 
 661   // Fast path: Reference is null (JNI tags are zero for null pointers).
 662   __ cmpdi(CCR0, obj, 0);
 663   __ beq(CCR0, done);
 664 
 665   // Resolve jobject using standard implementation.
 666   BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
 667 
 668   // Check whether heap is stable.
 669   __ lbz(tmp,
 670          in_bytes(ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()),
 671          jni_env);
 672 
 673   __ andi_(tmp, tmp, ShenandoahHeap::EVACUATION | ShenandoahHeap::HAS_FORWARDED);
 674   __ bne(CCR0, slowpath);
 675 
 676   __ bind(done);
 677   __ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
 678 }
 679 
 680 // Special shenandoah CAS implementation that handles false negatives due
 681 // to concurrent evacuation.  That is, the CAS operation is intended to succeed in
 682 // the following scenarios (success criteria):
 683 //  s1) The reference pointer ('base_addr') equals the expected ('expected') pointer.
 684 //  s2) The reference pointer refers to the from-space version of an already-evacuated
 685 //      object, whereas the expected pointer refers to the to-space version of the same object.
 686 // Situations in which the reference pointer refers to the to-space version of an object
 687 // and the expected pointer refers to the from-space version of the same object can not occur due to
 688 // shenandoah's strong to-space invariant.  This also implies that the reference stored in 'new_val'
 689 // can not refer to the from-space version of an already-evacuated object.
 690 //
 691 // To guarantee correct behavior in concurrent environments, two races must be addressed:
 692 //  r1) A concurrent thread may heal the reference pointer (i.e., it is no longer referring to the
 693 //      from-space version but to the to-space version of the object in question).
 694 //      In this case, the CAS operation should succeed.
 695 //  r2) A concurrent thread may mutate the reference (i.e., the reference pointer refers to an entirely different object).
 696 //      In this case, the CAS operation should fail.
 697 //
 698 // By default, the value held in the 'result' register is zero to indicate failure of CAS,
 699 // non-zero to indicate success.  If 'is_cae' is set, the result is the most recently fetched
 700 // value from 'base_addr' rather than a boolean success indicator.
 701 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler *masm, Register base_addr,
 702                                                 Register expected, Register new_val, Register tmp1, Register tmp2,
 703                                                 bool is_cae, Register result) {
 704   __ block_comment("cmpxchg_oop (shenandoahgc) {");
 705 
 706   assert_different_registers(base_addr, new_val, tmp1, tmp2, result, R0);
 707   assert_different_registers(base_addr, expected, tmp1, tmp2, result, R0);
 708 
 709   // Potential clash of 'success_flag' and 'tmp' is being accounted for.
 710   Register success_flag  = is_cae ? noreg  : result,
 711            current_value = is_cae ? result : tmp1,
 712            tmp           = is_cae ? tmp1   : result,
 713            initial_value = tmp2;
 714 
 715   Label done, step_four;
 716 
 717   __ bind(step_four);
 718 
 719   /* ==== Step 1 ("Standard" CAS) ==== */
 720   // Fast path: The values stored in 'expected' and 'base_addr' are equal.
 721   // Given that 'expected' must refer to the to-space object of an evacuated object (strong to-space invariant),
 722   // no special processing is required.
 723   if (UseCompressedOops) {
 724     __ cmpxchgw(CCR0, current_value, expected, new_val, base_addr, MacroAssembler::MemBarNone,
 725                 false, success_flag, true);
 726   } else {
 727     __ cmpxchgd(CCR0, current_value, expected, new_val, base_addr, MacroAssembler::MemBarNone,
 728                 false, success_flag, NULL, true);
 729   }
 730 
 731   // Skip the rest of the barrier if the CAS operation succeeds immediately.
 732   // If it does not, the value stored at the address is either the from-space pointer of the
 733   // referenced object (success criteria s2)) or simply another object.
 734   __ beq(CCR0, done);
 735 
 736   /* ==== Step 2 (Null check) ==== */
 737   // The success criteria s2) cannot be matched with a null pointer
 738   // (null pointers cannot be subject to concurrent evacuation).  The failure of the CAS operation is thus legitimate.
 739   __ cmpdi(CCR0, current_value, 0);
 740   __ beq(CCR0, done);
 741 
 742   /* ==== Step 3 (reference pointer refers to from-space version; success criteria s2)) ==== */
 743   // To check whether the reference pointer refers to the from-space version, the forward
 744   // pointer of the object referred to by the reference is resolved and compared against the expected pointer.
 745   // If this check succeed, another CAS operation is issued with the from-space pointer being the expected pointer.
 746   //
 747   // Save the potential from-space pointer.
 748   __ mr(initial_value, current_value);
 749 
 750   // Resolve forward pointer.
 751   if (UseCompressedOops) { __ decode_heap_oop_not_null(current_value); }
 752   resolve_forward_pointer_not_null(masm, current_value, tmp);
 753   if (UseCompressedOops) { __ encode_heap_oop_not_null(current_value); }
 754 
 755   if (!is_cae) {
 756     // 'success_flag' was overwritten by call to 'resovle_forward_pointer_not_null'.
 757     // Load zero into register for the potential failure case.
 758     __ li(success_flag, 0);
 759   }
 760   __ cmpd(CCR0, current_value, expected);
 761   __ bne(CCR0, done);
 762 
 763   // Discard fetched value as it might be a reference to the from-space version of an object.
 764   if (UseCompressedOops) {
 765     __ cmpxchgw(CCR0, R0, initial_value, new_val, base_addr, MacroAssembler::MemBarNone,
 766                 false, success_flag);
 767   } else {
 768     __ cmpxchgd(CCR0, R0, initial_value, new_val, base_addr, MacroAssembler::MemBarNone,
 769                 false, success_flag);
 770   }
 771 
 772   /* ==== Step 4 (Retry CAS with to-space pointer (success criteria s2) under race r1)) ==== */
 773   // The reference pointer could have been healed whilst the previous CAS operation was being performed.
 774   // Another CAS operation must thus be issued with the to-space pointer being the expected pointer.
 775   // If that CAS operation fails as well, race r2) must have occurred, indicating that
 776   // the operation failure is legitimate.
 777   //
 778   // To keep the code's size small and thus improving cache (icache) performance, this highly
 779   // unlikely case should be handled by the smallest possible code.  Instead of emitting a third,
 780   // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
 781   // (passed arguments are identical).
 782   //
 783   // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
 784   // to fail.  Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
 785   // It is thus important to ensure that a re-execution of those steps does not put program correctness
 786   // at risk:
 787   // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
 788   // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
 789   //           fails.  Unless the reference has been updated in the meanwhile once again, this is
 790   //           guaranteed to be the case.
 791   //           In case of a concurrent update, the CAS would be retried again. This is legitimate
 792   //           in terms of program correctness (even though it is not desired).
 793   __ bne(CCR0, step_four);
 794 
 795   __ bind(done);
 796   __ block_comment("} cmpxchg_oop (shenandoahgc)");
 797 }
 798 
 799 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 800                                                                      Register addr, Register count, Register preserve) {
 801   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 802     return;
 803   }
 804 
 805   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
 806   CardTable* ct = bs->card_table();
 807   assert_different_registers(addr, count, R0);
 808 
 809   Label Lskip_loop, Lstore_loop;
 810 
 811   __ sldi_(count, count, LogBytesPerHeapOop);
 812   __ beq(CCR0, Lskip_loop); // zero length
 813   __ addi(count, count, -BytesPerHeapOop);
 814   __ add(count, addr, count);
 815   // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
 816   __ srdi(addr, addr, CardTable::card_shift());
 817   __ srdi(count, count, CardTable::card_shift());
 818   __ subf(count, addr, count);
 819   __ add_const_optimized(addr, addr, (address)ct->byte_map_base(), R0);
 820   __ addi(count, count, 1);
 821   __ li(R0, 0);
 822   __ mtctr(count);
 823   // Byte store loop
 824   __ bind(Lstore_loop);
 825   __ stb(R0, 0, addr);
 826   __ addi(addr, addr, 1);
 827   __ bdnz(Lstore_loop);
 828   __ bind(Lskip_loop);
 829 }
 830 
 831 #undef __
 832 
 833 #ifdef COMPILER1
 834 
 835 #define __ ce->masm()->
 836 
 837 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
 838   __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
 839 
 840   ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
 841   __ bind(*stub->entry());
 842 
 843   // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
 844   // This stub is the slowpath of that function.
 845 
 846   assert(stub->pre_val()->is_register(), "pre_val must be a register");
 847   Register pre_val = stub->pre_val()->as_register();
 848 
 849   // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
 850   // ("preloaded mode" of the store barrier).
 851   if (stub->do_load()) {
 852     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false);
 853   }
 854 
 855   // Fast path: Reference is null.
 856   __ cmpdi(CCR0, pre_val, 0);
 857   __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CCR0, Assembler::equal), *stub->continuation());
 858 
 859   // Argument passing via the stack.
 860   __ std(pre_val, -8, R1_SP);
 861 
 862   __ load_const_optimized(R0, bs->pre_barrier_c1_runtime_code_blob()->code_begin());
 863   __ call_stub(R0);
 864 
 865   __ b(*stub->continuation());
 866   __ block_comment("} gen_pre_barrier_stub (shenandoahgc)");
 867 }
 868 
 869 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler *ce,
 870                                                                     ShenandoahLoadReferenceBarrierStub *stub) {
 871   __ block_comment("gen_load_reference_barrier_stub (shenandoahgc) {");
 872 
 873   ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
 874   __ bind(*stub->entry());
 875 
 876   Register obj  = stub->obj()->as_register();
 877   Register res  = stub->result()->as_register();
 878   Register addr = stub->addr()->as_pointer_register();
 879   Register tmp1 = stub->tmp1()->as_register();
 880   Register tmp2 = stub->tmp2()->as_register();
 881   assert_different_registers(addr, res, tmp1, tmp2);
 882 
 883 #ifdef ASSERT
 884   // Ensure that 'res' is 'R3_ARG1' and contains the same value as 'obj' to reduce the number of required
 885   // copy instructions.
 886   assert(R3_RET == res, "res must be r3");
 887   __ cmpd(CCR0, res, obj);
 888   __ asm_assert_eq("result register must contain the reference stored in obj");
 889 #endif
 890 
 891   DecoratorSet decorators = stub->decorators();
 892 
 893   /* ==== Check whether region is in collection set ==== */
 894   // GC status (unstable) has already been verified by 'ShenandoahBarrierSetC1::load_reference_barrier_impl'.
 895   // This stub is the slowpath of that function.
 896 
 897   bool is_strong  = ShenandoahBarrierSet::is_strong_access(decorators);
 898   bool is_weak    = ShenandoahBarrierSet::is_weak_access(decorators);
 899   bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
 900   bool is_native  = ShenandoahBarrierSet::is_native_access(decorators);
 901 
 902   if (is_strong) {
 903     // Check whether object is in collection set.
 904     __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
 905     __ srdi(tmp1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
 906     __ lbzx(tmp2, tmp1, tmp2);
 907 
 908     __ andi_(tmp2, tmp2, 1);
 909     __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CCR0, Assembler::equal), *stub->continuation());
 910   }
 911 
 912   address blob_addr = nullptr;
 913 
 914   if (is_strong) {
 915     if (is_native) {
 916       blob_addr = bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin();
 917     } else {
 918       blob_addr = bs->load_reference_barrier_strong_rt_code_blob()->code_begin();
 919     }
 920   } else if (is_weak) {
 921     blob_addr = bs->load_reference_barrier_weak_rt_code_blob()->code_begin();
 922   } else {
 923     assert(is_phantom, "only remaining strength");
 924     blob_addr = bs->load_reference_barrier_phantom_rt_code_blob()->code_begin();
 925   }
 926 
 927   assert(blob_addr != nullptr, "code blob cannot be found");
 928 
 929   // Argument passing via the stack.  'obj' is passed implicitly (as asserted above).
 930   __ std(addr, -8, R1_SP);
 931 
 932   __ load_const_optimized(tmp1, blob_addr, tmp2);
 933   __ call_stub(tmp1);
 934 
 935   // 'res' is 'R3_RET'.  The result is thus already in the correct register.
 936 
 937   __ b(*stub->continuation());
 938   __ block_comment("} gen_load_reference_barrier_stub (shenandoahgc)");
 939 }
 940 
 941 #undef __
 942 
 943 #define __ sasm->
 944 
 945 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler *sasm) {
 946   __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
 947 
 948   Label runtime, skip_barrier;
 949   BarrierSet *bs = BarrierSet::barrier_set();
 950 
 951   // Argument passing via the stack.
 952   const int caller_stack_slots = 3;
 953 
 954   Register R0_pre_val = R0;
 955   __ ld(R0, -8, R1_SP);
 956   Register R11_tmp1 = R11_scratch1;
 957   __ std(R11_tmp1, -16, R1_SP);
 958   Register R12_tmp2 = R12_scratch2;
 959   __ std(R12_tmp2, -24, R1_SP);
 960 
 961   /* ==== Check whether marking is active ==== */
 962   // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
 963   // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
 964   __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 965 
 966   __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
 967   __ beq(CCR0, skip_barrier);
 968 
 969   /* ==== Add previous value directly to thread-local SATB mark queue ==== */
 970   // Check queue's capacity.  Jump to runtime if no free slot is available.
 971   __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 972   __ cmpdi(CCR0, R12_tmp2, 0);
 973   __ beq(CCR0, runtime);
 974 
 975   // Capacity suffices.  Decrement the queue's size by one slot (size of one oop).
 976   __ addi(R12_tmp2, R12_tmp2, -wordSize);
 977   __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 978 
 979   // Enqueue the previous value and skip the runtime invocation.
 980   __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
 981   __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
 982   __ b(skip_barrier);
 983 
 984   __ bind(runtime);
 985 
 986   /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
 987   // Save to-be-preserved registers.
 988   const int nbytes_save = (MacroAssembler::num_volatile_regs + caller_stack_slots) * BytesPerWord;
 989   __ save_volatile_gprs(R1_SP, -nbytes_save);
 990   __ save_LR_CR(R11_tmp1);
 991   __ push_frame_reg_args(nbytes_save, R11_tmp1);
 992 
 993   // Invoke runtime.
 994   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), R0_pre_val, R16_thread);
 995 
 996   // Restore to-be-preserved registers.
 997   __ pop_frame();
 998   __ restore_LR_CR(R11_tmp1);
 999   __ restore_volatile_gprs(R1_SP, -nbytes_save);
1000 
1001   __ bind(skip_barrier);
1002 
1003   // Restore spilled registers.
1004   __ ld(R11_tmp1, -16, R1_SP);
1005   __ ld(R12_tmp2, -24, R1_SP);
1006 
1007   __ blr();
1008   __ block_comment("} generate_c1_pre_barrier_runtime_stub (shenandoahgc)");
1009 }
1010 
1011 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler *sasm,
1012                                                                                     DecoratorSet decorators) {
1013   __ block_comment("generate_c1_load_reference_barrier_runtime_stub (shenandoahgc) {");
1014 
1015   // Argument passing via the stack.
1016   const int caller_stack_slots = 1;
1017 
1018   // Save to-be-preserved registers.
1019   const int nbytes_save = (MacroAssembler::num_volatile_regs - 1 // 'R3_ARG1' is skipped
1020                            + caller_stack_slots) * BytesPerWord;
1021   __ save_volatile_gprs(R1_SP, -nbytes_save, true, false);
1022 
1023   // Load arguments from stack.
1024   // No load required, as assured by assertions in 'ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub'.
1025   Register R3_obj = R3_ARG1;
1026   Register R4_load_addr = R4_ARG2;
1027   __ ld(R4_load_addr, -8, R1_SP);
1028 
1029   Register R11_tmp = R11_scratch1;
1030 
1031   /* ==== Invoke runtime ==== */
1032   bool is_strong  = ShenandoahBarrierSet::is_strong_access(decorators);
1033   bool is_weak    = ShenandoahBarrierSet::is_weak_access(decorators);
1034   bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
1035   bool is_native  = ShenandoahBarrierSet::is_native_access(decorators);
1036 
1037   address jrt_address = NULL;
1038 
1039   if (is_strong) {
1040     if (is_native) {
1041       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1042     } else {
1043       if (UseCompressedOops) {
1044         jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
1045       } else {
1046         jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1047       }
1048     }
1049   } else if (is_weak) {
1050     assert(!is_native, "weak load reference barrier must not be called off-heap");
1051     if (UseCompressedOops) {
1052       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
1053     } else {
1054       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
1055     }
1056   } else {
1057     assert(is_phantom, "reference type must be phantom");
1058     assert(is_native, "phantom load reference barrier must be called off-heap");
1059     jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
1060   }
1061   assert(jrt_address != NULL, "load reference barrier runtime routine cannot be found");
1062 
1063   __ save_LR_CR(R11_tmp);
1064   __ push_frame_reg_args(nbytes_save, R11_tmp);
1065 
1066   // Invoke runtime.  Arguments are already stored in the corresponding registers.
1067   __ call_VM_leaf(jrt_address, R3_obj, R4_load_addr);
1068 
1069   // Restore to-be-preserved registers.
1070   __ pop_frame();
1071   __ restore_LR_CR(R11_tmp);
1072   __ restore_volatile_gprs(R1_SP, -nbytes_save, true, false); // Skip 'R3_RET' register.
1073 
1074   __ blr();
1075   __ block_comment("} generate_c1_load_reference_barrier_runtime_stub (shenandoahgc)");
1076 }
1077 
1078 #undef __
1079 
1080 #endif // COMPILER1