1 /*
   2  * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
   3  * Copyright (c) 2012, 2021 SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "gc/shared/gcArguments.hpp"
  27 #include "gc/shared/gc_globals.hpp"
  28 #include "macroAssembler_ppc.hpp"
  29 #include "precompiled.hpp"
  30 #include "asm/macroAssembler.inline.hpp"
  31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  32 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  33 #include "gc/shenandoah/shenandoahForwarding.hpp"
  34 #include "gc/shenandoah/shenandoahHeap.hpp"
  35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  37 #include "gc/shenandoah/shenandoahRuntime.hpp"
  38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  39 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  40 #include "interpreter/interpreter.hpp"
  41 #include "runtime/javaThread.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "utilities/globalDefinitions.hpp"
  44 #include "vm_version_ppc.hpp"
  45 #ifdef COMPILER1
  46 #include "c1/c1_LIRAssembler.hpp"
  47 #include "c1/c1_MacroAssembler.hpp"
  48 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  49 #endif
  50 
  51 #define __ masm->
  52 
  53 void ShenandoahBarrierSetAssembler::satb_write_barrier(MacroAssembler *masm,
  54                                                        Register base, RegisterOrConstant ind_or_offs,
  55                                                        Register tmp1, Register tmp2, Register tmp3,
  56                                                        MacroAssembler::PreservationLevel preservation_level) {
  57   if (ShenandoahSATBBarrier) {
  58     __ block_comment("satb_write_barrier (shenandoahgc) {");
  59     satb_write_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
  60     __ block_comment("} satb_write_barrier (shenandoahgc)");
  61   }
  62 }
  63 
  64 void ShenandoahBarrierSetAssembler::iu_barrier(MacroAssembler *masm,
  65                                                Register val,
  66                                                Register tmp1, Register tmp2,
  67                                                MacroAssembler::PreservationLevel preservation_level,
  68                                                DecoratorSet decorators) {
  69   // IU barriers are also employed to avoid resurrection of weak references,
  70   // even if Shenandoah does not operate in incremental update mode.
  71   if (ShenandoahIUBarrier || ShenandoahSATBBarrier) {
  72     __ block_comment("iu_barrier (shenandoahgc) {");
  73     satb_write_barrier_impl(masm, decorators, noreg, noreg, val, tmp1, tmp2, preservation_level);
  74     __ block_comment("} iu_barrier (shenandoahgc)");
  75   }
  76 }
  77 
  78 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
  79                                                            Register base, RegisterOrConstant ind_or_offs,
  80                                                            Register dst,
  81                                                            Register tmp1, Register tmp2,
  82                                                            MacroAssembler::PreservationLevel preservation_level) {
  83   if (ShenandoahLoadRefBarrier) {
  84     __ block_comment("load_reference_barrier (shenandoahgc) {");
  85     load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
  86     __ block_comment("} load_reference_barrier (shenandoahgc)");
  87   }
  88 }
  89 
  90 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
  91                                                        Register src, Register dst, Register count,
  92                                                        Register preserve1, Register preserve2) {
  93   __ block_comment("arraycopy_prologue (shenandoahgc) {");
  94 
  95   Register R11_tmp = R11_scratch1;
  96 
  97   assert_different_registers(src, dst, count, R11_tmp, noreg);
  98   if (preserve1 != noreg) {
  99     // Technically not required, but likely to indicate an error.
 100     assert_different_registers(preserve1, preserve2);
 101   }
 102 
 103   /* ==== Check whether barrier is required (optimizations) ==== */
 104   // Fast path: Component type of array is not a reference type.
 105   if (!is_reference_type(type)) {
 106     return;
 107   }
 108 
 109   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 110 
 111   // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
 112   // any useful information.
 113   if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahIUBarrier && !ShenandoahLoadRefBarrier) {
 114     return;
 115   }
 116 
 117   Label skip_prologue;
 118 
 119   // Fast path: Array is of length zero.
 120   __ cmpdi(CCR0, count, 0);
 121   __ beq(CCR0, skip_prologue);
 122 
 123   /* ==== Check whether barrier is required (gc state) ==== */
 124   __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
 125          R16_thread);
 126 
 127   // The set of garbage collection states requiring barriers depends on the available barrier types and the
 128   // type of the reference in question.
 129   // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
 130   // for the garbage collector.
 131   const int required_states = ShenandoahSATBBarrier && dest_uninitialized
 132                               ? ShenandoahHeap::HAS_FORWARDED
 133                               : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
 134 
 135   __ andi_(R11_tmp, R11_tmp, required_states);
 136   __ beq(CCR0, skip_prologue);
 137 
 138   /* ==== Invoke runtime ==== */
 139   // Save to-be-preserved registers.
 140   int highest_preserve_register_index = 0;
 141   {
 142     if (preserve1 != noreg && preserve1->is_volatile()) {
 143       __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 144     }
 145     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 146       __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 147     }
 148 
 149     __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 150     __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 151     __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 152 
 153     __ save_LR_CR(R11_tmp);
 154     __ push_frame_reg_args(-BytesPerWord * highest_preserve_register_index,
 155                            R11_tmp);
 156   }
 157 
 158   // Invoke runtime.
 159   address jrt_address = nullptr;
 160   if (UseCompressedOops) {
 161     jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry);
 162   } else {
 163     jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry);
 164   }
 165   assert(jrt_address != nullptr, "jrt routine cannot be found");
 166 
 167   __ call_VM_leaf(jrt_address, src, dst, count);
 168 
 169   // Restore to-be-preserved registers.
 170   {
 171     __ pop_frame();
 172     __ restore_LR_CR(R11_tmp);
 173 
 174     __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 175     __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 176     __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 177 
 178     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 179       __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 180     }
 181     if (preserve1 != noreg && preserve1->is_volatile()) {
 182       __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 183     }
 184   }
 185 
 186   __ bind(skip_prologue);
 187   __ block_comment("} arraycopy_prologue (shenandoahgc)");
 188 }
 189 
 190 // The to-be-enqueued value can either be determined
 191 // - dynamically by passing the reference's address information (load mode) or
 192 // - statically by passing a register the value is stored in (preloaded mode)
 193 //   - for performance optimizations in cases where the previous value is known (currently not implemented) and
 194 //   - for incremental-update barriers.
 195 //
 196 // decorators:  The previous value's decorator set.
 197 //              In "load mode", the value must equal '0'.
 198 // base:        Base register of the reference's address (load mode).
 199 //              In "preloaded mode", the register must equal 'noreg'.
 200 // ind_or_offs: Index or offset of the reference's address (load mode).
 201 //              If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
 202 // pre_val:     Register holding the to-be-stored value (preloaded mode).
 203 //              In "load mode", this register acts as a temporary register and must
 204 //              thus not be 'noreg'.  In "preloaded mode", its content will be sustained.
 205 // tmp1/tmp2:   Temporary registers, one of which must be non-volatile in "preloaded mode".
 206 void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
 207                                                             Register base, RegisterOrConstant ind_or_offs,
 208                                                             Register pre_val,
 209                                                             Register tmp1, Register tmp2,
 210                                                             MacroAssembler::PreservationLevel preservation_level) {
 211   assert_different_registers(tmp1, tmp2, pre_val, noreg);
 212 
 213   Label skip_barrier;
 214 
 215   /* ==== Determine necessary runtime invocation preservation measures ==== */
 216   const bool needs_frame           = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
 217   const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
 218   const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
 219 
 220   // Check whether marking is active.
 221   __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 222 
 223   __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
 224   __ beq(CCR0, skip_barrier);
 225 
 226   /* ==== Determine the reference's previous value ==== */
 227   bool preloaded_mode = base == noreg;
 228   Register pre_val_save = noreg;
 229 
 230   if (preloaded_mode) {
 231     // Previous value has been passed to the method, so it must not be determined manually.
 232     // In case 'pre_val' is a volatile register, it must be saved across the C-call
 233     // as callers may depend on its value.
 234     // Unless the general purposes registers are saved anyway, one of the temporary registers
 235     // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
 236     if (!preserve_gp_registers && pre_val->is_volatile()) {
 237       pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
 238       assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
 239     }
 240 
 241     if ((decorators & IS_NOT_NULL) != 0) {
 242 #ifdef ASSERT
 243       __ cmpdi(CCR0, pre_val, 0);
 244       __ asm_assert_ne("null oop is not allowed");
 245 #endif // ASSERT
 246     } else {
 247       __ cmpdi(CCR0, pre_val, 0);
 248       __ beq(CCR0, skip_barrier);
 249     }
 250   } else {
 251     // Load from the reference address to determine the reference's current value (before the store is being performed).
 252     // Contrary to the given value in "preloaded mode", it is not necessary to preserve it.
 253     assert(decorators == 0, "decorator set must be empty");
 254     assert(base != noreg, "base must be a register");
 255     assert(!ind_or_offs.is_register() || ind_or_offs.as_register() != noreg, "ind_or_offs must be a register");
 256     if (UseCompressedOops) {
 257       __ lwz(pre_val, ind_or_offs, base);
 258     } else {
 259       __ ld(pre_val, ind_or_offs, base);
 260     }
 261 
 262     __ cmpdi(CCR0, pre_val, 0);
 263     __ beq(CCR0, skip_barrier);
 264 
 265     if (UseCompressedOops) {
 266       __ decode_heap_oop_not_null(pre_val);
 267     }
 268   }
 269 
 270   /* ==== Try to enqueue the to-be-stored value directly into thread's local SATB mark queue ==== */
 271   {
 272     Label runtime;
 273     Register Rbuffer = tmp1, Rindex = tmp2;
 274 
 275     // Check whether the queue has enough capacity to store another oop.
 276     // If not, jump to the runtime to commit the buffer and to allocate a new one.
 277     // (The buffer's index corresponds to the amount of remaining free space.)
 278     __ ld(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 279     __ cmpdi(CCR0, Rindex, 0);
 280     __ beq(CCR0, runtime); // If index == 0 (buffer is full), goto runtime.
 281 
 282     // Capacity suffices.  Decrement the queue's size by the size of one oop.
 283     // (The buffer is filled contrary to the heap's growing direction, i.e., it is filled downwards.)
 284     __ addi(Rindex, Rindex, -wordSize);
 285     __ std(Rindex, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 286 
 287     // Enqueue the previous value and skip the invocation of the runtime.
 288     __ ld(Rbuffer, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
 289     __ stdx(pre_val, Rbuffer, Rindex);
 290     __ b(skip_barrier);
 291 
 292     __ bind(runtime);
 293   }
 294 
 295   /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
 296   // Save to-be-preserved registers.
 297   int nbytes_save = 0;
 298 
 299   if (needs_frame) {
 300     if (preserve_gp_registers) {
 301       nbytes_save = (preserve_fp_registers
 302                      ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
 303                      : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
 304       __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
 305     }
 306 
 307     __ save_LR_CR(tmp1);
 308     __ push_frame_reg_args(nbytes_save, tmp2);
 309   }
 310 
 311   if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
 312     assert(pre_val_save != noreg, "nv_save must not be noreg");
 313 
 314     // 'pre_val' register must be saved manually unless general-purpose are preserved in general.
 315     __ mr(pre_val_save, pre_val);
 316   }
 317 
 318   // Invoke runtime.
 319   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, R16_thread);
 320 
 321   // Restore to-be-preserved registers.
 322   if (!preserve_gp_registers && preloaded_mode && pre_val->is_volatile()) {
 323     __ mr(pre_val, pre_val_save);
 324   }
 325 
 326   if (needs_frame) {
 327     __ pop_frame();
 328     __ restore_LR_CR(tmp1);
 329 
 330     if (preserve_gp_registers) {
 331       __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
 332     }
 333   }
 334 
 335   __ bind(skip_barrier);
 336 }
 337 
 338 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler *masm, Register dst, Register tmp) {
 339   __ block_comment("resolve_forward_pointer_not_null (shenandoahgc) {");
 340 
 341   Register tmp1 = tmp,
 342            R0_tmp2 = R0;
 343   assert_different_registers(dst, tmp1, R0_tmp2, noreg);
 344 
 345   // If the object has been evacuated, the mark word layout is as follows:
 346   // | forwarding pointer (62-bit) | '11' (2-bit) |
 347 
 348   // The invariant that stack/thread pointers have the lowest two bits cleared permits retrieving
 349   // the forwarding pointer solely by inversing the lowest two bits.
 350   // This invariant follows inevitably from hotspot's minimal alignment.
 351   assert(markWord::marked_value <= (unsigned long) MinObjAlignmentInBytes,
 352          "marked value must not be higher than hotspot's minimal alignment");
 353 
 354   Label done;
 355 
 356   // Load the object's mark word.
 357   __ ld(tmp1, oopDesc::mark_offset_in_bytes(), dst);
 358 
 359   // Load the bit mask for the lock bits.
 360   __ li(R0_tmp2, markWord::lock_mask_in_place);
 361 
 362   // Check whether all bits matching the bit mask are set.
 363   // If that is the case, the object has been evacuated and the most significant bits form the forward pointer.
 364   __ andc_(R0_tmp2, R0_tmp2, tmp1);
 365 
 366   assert(markWord::lock_mask_in_place == markWord::marked_value,
 367          "marked value must equal the value obtained when all lock bits are being set");
 368   if (VM_Version::has_isel()) {
 369     __ xori(tmp1, tmp1, markWord::lock_mask_in_place);
 370     __ isel(dst, CCR0, Assembler::equal, false, tmp1);
 371   } else {
 372     __ bne(CCR0, done);
 373     __ xori(dst, tmp1, markWord::lock_mask_in_place);
 374   }
 375 
 376   __ bind(done);
 377   __ block_comment("} resolve_forward_pointer_not_null (shenandoahgc)");
 378 }
 379 
 380 // base:        Base register of the reference's address.
 381 // ind_or_offs: Index or offset of the reference's address (load mode).
 382 // dst:         Reference's address.  In case the object has been evacuated, this is the to-space version
 383 //              of that object.
 384 void ShenandoahBarrierSetAssembler::load_reference_barrier_impl(
 385     MacroAssembler *masm, DecoratorSet decorators,
 386     Register base, RegisterOrConstant ind_or_offs,
 387     Register dst,
 388     Register tmp1, Register tmp2,
 389     MacroAssembler::PreservationLevel preservation_level) {
 390   if (ind_or_offs.is_register()) {
 391     assert_different_registers(tmp1, tmp2, base, ind_or_offs.as_register(), dst, noreg);
 392   } else {
 393     assert_different_registers(tmp1, tmp2, base, dst, noreg);
 394   }
 395 
 396   Label skip_barrier;
 397 
 398   bool is_strong  = ShenandoahBarrierSet::is_strong_access(decorators);
 399   bool is_weak    = ShenandoahBarrierSet::is_weak_access(decorators);
 400   bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
 401   bool is_native  = ShenandoahBarrierSet::is_native_access(decorators);
 402   bool is_narrow  = UseCompressedOops && !is_native;
 403 
 404   /* ==== Check whether heap is stable ==== */
 405   __ lbz(tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 406 
 407   if (is_strong) {
 408     // For strong references, the heap is considered stable if "has forwarded" is not active.
 409     __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
 410     __ beq(CCR0, skip_barrier);
 411 #ifdef ASSERT
 412     // "evacuation" -> (implies) "has forwarded".  If we reach this code, "has forwarded" must thus be set.
 413     __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
 414     __ asm_assert_ne("'has forwarded' is missing");
 415 #endif // ASSERT
 416   } else {
 417     // For all non-strong references, the heap is considered stable if not any of "has forwarded",
 418     // "root set processing", and "weak reference processing" is active.
 419     // The additional phase conditions are in place to avoid the resurrection of weak references (see JDK-8266440).
 420     Label skip_fastpath;
 421     __ andi_(tmp1, tmp2, ShenandoahHeap::WEAK_ROOTS);
 422     __ bne(CCR0, skip_fastpath);
 423 
 424     __ andi_(tmp1, tmp2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
 425     __ beq(CCR0, skip_barrier);
 426 #ifdef ASSERT
 427     // "evacuation" -> (implies) "has forwarded".  If we reach this code, "has forwarded" must thus be set.
 428     __ andi_(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED);
 429     __ asm_assert_ne("'has forwarded' is missing");
 430 #endif // ASSERT
 431 
 432     __ bind(skip_fastpath);
 433   }
 434 
 435   /* ==== Check whether region is in collection set ==== */
 436   if (is_strong) {
 437     // Shenandoah stores metadata on regions in a continuous area of memory in which a single byte corresponds to
 438     // an entire region of the shenandoah heap.  At present, only the least significant bit is of significance
 439     // and indicates whether the region is part of the collection set.
 440     //
 441     // All regions are of the same size and are always aligned by a power of two.
 442     // Any address can thus be shifted by a fixed number of bits to retrieve the address prefix shared by
 443     // all objects within that region (region identification bits).
 444     //
 445     //  | unused bits | region identification bits | object identification bits |
 446     //  (Region size depends on a couple of criteria, such as page size, user-provided arguments and the max heap size.
 447     //   The number of object identification bits can thus not be determined at compile time.)
 448     //
 449     // -------------------------------------------------------  <--- cs (collection set) base address
 450     // | lost space due to heap space base address                   -> 'ShenandoahHeap::in_cset_fast_test_addr()'
 451     // | (region identification bits contain heap base offset)
 452     // |------------------------------------------------------  <--- cs base address + (heap_base >> region size shift)
 453     // | collection set in the proper                                -> shift: 'region_size_bytes_shift_jint()'
 454     // |
 455     // |------------------------------------------------------  <--- cs base address + (heap_base >> region size shift)
 456     //                                                                               + number of regions
 457     __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
 458     __ srdi(tmp1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
 459     __ lbzx(tmp2, tmp1, tmp2);
 460     __ andi_(tmp2, tmp2, 1);
 461     __ beq(CCR0, skip_barrier);
 462   }
 463 
 464   /* ==== Invoke runtime ==== */
 465   // Save to-be-preserved registers.
 466   int nbytes_save = 0;
 467 
 468   const bool needs_frame           = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
 469   const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
 470   const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
 471 
 472   if (needs_frame) {
 473     if (preserve_gp_registers) {
 474       nbytes_save = (preserve_fp_registers
 475                      ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
 476                      : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
 477       __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
 478     }
 479 
 480     __ save_LR_CR(tmp1);
 481     __ push_frame_reg_args(nbytes_save, tmp1);
 482   }
 483 
 484   // Calculate the reference's absolute address.
 485   __ add(R4_ARG2, ind_or_offs, base);
 486 
 487   // Invoke runtime.
 488   address jrt_address = nullptr;
 489 
 490   if (is_strong) {
 491     if (is_narrow) {
 492       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
 493     } else {
 494       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
 495     }
 496   } else if (is_weak) {
 497     if (is_narrow) {
 498       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
 499     } else {
 500       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
 501     }
 502   } else {
 503     assert(is_phantom, "only remaining strength");
 504     assert(!is_narrow, "phantom access cannot be narrow");
 505     jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
 506   }
 507   assert(jrt_address != nullptr, "jrt routine cannot be found");
 508 
 509   __ call_VM_leaf(jrt_address, dst /* reference */, R4_ARG2 /* reference address */);
 510 
 511   // Restore to-be-preserved registers.
 512   if (preserve_gp_registers) {
 513     __ mr(R0, R3_RET);
 514   } else {
 515     __ mr_if_needed(dst, R3_RET);
 516   }
 517 
 518   if (needs_frame) {
 519     __ pop_frame();
 520     __ restore_LR_CR(tmp1);
 521 
 522     if (preserve_gp_registers) {
 523       __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers);
 524       __ mr(dst, R0);
 525     }
 526   }
 527 
 528   __ bind(skip_barrier);
 529 }
 530 
 531 // base:           Base register of the reference's address.
 532 // ind_or_offs:    Index or offset of the reference's address.
 533 // L_handle_null:  An optional label that will be jumped to if the reference is null.
 534 void ShenandoahBarrierSetAssembler::load_at(
 535     MacroAssembler *masm, DecoratorSet decorators, BasicType type,
 536     Register base, RegisterOrConstant ind_or_offs, Register dst,
 537     Register tmp1, Register tmp2,
 538     MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
 539   // Register must not clash, except 'base' and 'dst'.
 540   if (ind_or_offs.is_register()) {
 541     if (base != noreg) {
 542       assert_different_registers(tmp1, tmp2, base, ind_or_offs.register_or_noreg(), R0, noreg);
 543     }
 544     assert_different_registers(tmp1, tmp2, dst, ind_or_offs.register_or_noreg(), R0, noreg);
 545   } else {
 546     if (base == noreg) {
 547       assert_different_registers(tmp1, tmp2, base, R0, noreg);
 548     }
 549     assert_different_registers(tmp1, tmp2, dst, R0, noreg);
 550   }
 551 
 552   /* ==== Apply load barrier, if required ==== */
 553   if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
 554     assert(is_reference_type(type), "need_load_reference_barrier must check whether type is a reference type");
 555 
 556     // If 'dst' clashes with either 'base' or 'ind_or_offs', use an intermediate result register
 557     // to keep the values of those alive until the load reference barrier is applied.
 558     Register intermediate_dst = (dst == base || (ind_or_offs.is_register() && dst == ind_or_offs.as_register()))
 559                                 ? tmp2
 560                                 : dst;
 561 
 562     BarrierSetAssembler::load_at(masm, decorators, type,
 563                                  base, ind_or_offs,
 564                                  intermediate_dst,
 565                                  tmp1, noreg,
 566                                  preservation_level, L_handle_null);
 567 
 568     load_reference_barrier(masm, decorators,
 569                            base, ind_or_offs,
 570                            intermediate_dst,
 571                            tmp1, R0,
 572                            preservation_level);
 573 
 574     __ mr_if_needed(dst, intermediate_dst);
 575   } else {
 576     BarrierSetAssembler::load_at(masm, decorators, type,
 577                                  base, ind_or_offs,
 578                                  dst,
 579                                  tmp1, tmp2,
 580                                  preservation_level, L_handle_null);
 581   }
 582 
 583   /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
 584   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
 585     iu_barrier(masm, dst, tmp1, tmp2, preservation_level);
 586   }
 587 }
 588 
 589 // base:        Base register of the reference's address.
 590 // ind_or_offs: Index or offset of the reference's address.
 591 // val:         To-be-stored value/reference's new value.
 592 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
 593                                              Register base, RegisterOrConstant ind_or_offs, Register val,
 594                                              Register tmp1, Register tmp2, Register tmp3,
 595                                              MacroAssembler::PreservationLevel preservation_level) {
 596   if (is_reference_type(type)) {
 597     if (ShenandoahSATBBarrier) {
 598       satb_write_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
 599     }
 600 
 601     if (ShenandoahIUBarrier && val != noreg) {
 602       iu_barrier(masm, val, tmp1, tmp2, preservation_level, decorators);
 603     }
 604   }
 605 
 606   BarrierSetAssembler::store_at(masm, decorators, type,
 607                                 base, ind_or_offs,
 608                                 val,
 609                                 tmp1, tmp2, tmp3,
 610                                 preservation_level);
 611 }
 612 
 613 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
 614                                                                   Register dst, Register jni_env, Register obj,
 615                                                                   Register tmp, Label &slowpath) {
 616   __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
 617 
 618   assert_different_registers(jni_env, obj, tmp);
 619 
 620   Label done;
 621 
 622   // Fast path: Reference is null (JNI tags are zero for null pointers).
 623   __ cmpdi(CCR0, obj, 0);
 624   __ beq(CCR0, done);
 625 
 626   // Resolve jobject using standard implementation.
 627   BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
 628 
 629   // Check whether heap is stable.
 630   __ lbz(tmp,
 631          in_bytes(ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()),
 632          jni_env);
 633 
 634   __ andi_(tmp, tmp, ShenandoahHeap::EVACUATION | ShenandoahHeap::HAS_FORWARDED);
 635   __ bne(CCR0, slowpath);
 636 
 637   __ bind(done);
 638   __ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
 639 }
 640 
 641 // Special shenandoah CAS implementation that handles false negatives due
 642 // to concurrent evacuation.  That is, the CAS operation is intended to succeed in
 643 // the following scenarios (success criteria):
 644 //  s1) The reference pointer ('base_addr') equals the expected ('expected') pointer.
 645 //  s2) The reference pointer refers to the from-space version of an already-evacuated
 646 //      object, whereas the expected pointer refers to the to-space version of the same object.
 647 // Situations in which the reference pointer refers to the to-space version of an object
 648 // and the expected pointer refers to the from-space version of the same object can not occur due to
 649 // shenandoah's strong to-space invariant.  This also implies that the reference stored in 'new_val'
 650 // can not refer to the from-space version of an already-evacuated object.
 651 //
 652 // To guarantee correct behavior in concurrent environments, two races must be addressed:
 653 //  r1) A concurrent thread may heal the reference pointer (i.e., it is no longer referring to the
 654 //      from-space version but to the to-space version of the object in question).
 655 //      In this case, the CAS operation should succeed.
 656 //  r2) A concurrent thread may mutate the reference (i.e., the reference pointer refers to an entirely different object).
 657 //      In this case, the CAS operation should fail.
 658 //
 659 // By default, the value held in the 'result' register is zero to indicate failure of CAS,
 660 // non-zero to indicate success.  If 'is_cae' is set, the result is the most recently fetched
 661 // value from 'base_addr' rather than a boolean success indicator.
 662 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler *masm, Register base_addr,
 663                                                 Register expected, Register new_val, Register tmp1, Register tmp2,
 664                                                 bool is_cae, Register result) {
 665   __ block_comment("cmpxchg_oop (shenandoahgc) {");
 666 
 667   assert_different_registers(base_addr, new_val, tmp1, tmp2, result, R0);
 668   assert_different_registers(base_addr, expected, tmp1, tmp2, result, R0);
 669 
 670   // Potential clash of 'success_flag' and 'tmp' is being accounted for.
 671   Register success_flag  = is_cae ? noreg  : result,
 672            current_value = is_cae ? result : tmp1,
 673            tmp           = is_cae ? tmp1   : result,
 674            initial_value = tmp2;
 675 
 676   Label done, step_four;
 677 
 678   __ bind(step_four);
 679 
 680   /* ==== Step 1 ("Standard" CAS) ==== */
 681   // Fast path: The values stored in 'expected' and 'base_addr' are equal.
 682   // Given that 'expected' must refer to the to-space object of an evacuated object (strong to-space invariant),
 683   // no special processing is required.
 684   if (UseCompressedOops) {
 685     __ cmpxchgw(CCR0, current_value, expected, new_val, base_addr, MacroAssembler::MemBarNone,
 686                 false, success_flag, true);
 687   } else {
 688     __ cmpxchgd(CCR0, current_value, expected, new_val, base_addr, MacroAssembler::MemBarNone,
 689                 false, success_flag, nullptr, true);
 690   }
 691 
 692   // Skip the rest of the barrier if the CAS operation succeeds immediately.
 693   // If it does not, the value stored at the address is either the from-space pointer of the
 694   // referenced object (success criteria s2)) or simply another object.
 695   __ beq(CCR0, done);
 696 
 697   /* ==== Step 2 (Null check) ==== */
 698   // The success criteria s2) cannot be matched with a null pointer
 699   // (null pointers cannot be subject to concurrent evacuation).  The failure of the CAS operation is thus legitimate.
 700   __ cmpdi(CCR0, current_value, 0);
 701   __ beq(CCR0, done);
 702 
 703   /* ==== Step 3 (reference pointer refers to from-space version; success criteria s2)) ==== */
 704   // To check whether the reference pointer refers to the from-space version, the forward
 705   // pointer of the object referred to by the reference is resolved and compared against the expected pointer.
 706   // If this check succeed, another CAS operation is issued with the from-space pointer being the expected pointer.
 707   //
 708   // Save the potential from-space pointer.
 709   __ mr(initial_value, current_value);
 710 
 711   // Resolve forward pointer.
 712   if (UseCompressedOops) { __ decode_heap_oop_not_null(current_value); }
 713   resolve_forward_pointer_not_null(masm, current_value, tmp);
 714   if (UseCompressedOops) { __ encode_heap_oop_not_null(current_value); }
 715 
 716   if (!is_cae) {
 717     // 'success_flag' was overwritten by call to 'resovle_forward_pointer_not_null'.
 718     // Load zero into register for the potential failure case.
 719     __ li(success_flag, 0);
 720   }
 721   __ cmpd(CCR0, current_value, expected);
 722   __ bne(CCR0, done);
 723 
 724   // Discard fetched value as it might be a reference to the from-space version of an object.
 725   if (UseCompressedOops) {
 726     __ cmpxchgw(CCR0, R0, initial_value, new_val, base_addr, MacroAssembler::MemBarNone,
 727                 false, success_flag);
 728   } else {
 729     __ cmpxchgd(CCR0, R0, initial_value, new_val, base_addr, MacroAssembler::MemBarNone,
 730                 false, success_flag);
 731   }
 732 
 733   /* ==== Step 4 (Retry CAS with to-space pointer (success criteria s2) under race r1)) ==== */
 734   // The reference pointer could have been healed whilst the previous CAS operation was being performed.
 735   // Another CAS operation must thus be issued with the to-space pointer being the expected pointer.
 736   // If that CAS operation fails as well, race r2) must have occurred, indicating that
 737   // the operation failure is legitimate.
 738   //
 739   // To keep the code's size small and thus improving cache (icache) performance, this highly
 740   // unlikely case should be handled by the smallest possible code.  Instead of emitting a third,
 741   // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
 742   // (passed arguments are identical).
 743   //
 744   // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
 745   // to fail.  Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
 746   // It is thus important to ensure that a re-execution of those steps does not put program correctness
 747   // at risk:
 748   // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
 749   // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
 750   //           fails.  Unless the reference has been updated in the meanwhile once again, this is
 751   //           guaranteed to be the case.
 752   //           In case of a concurrent update, the CAS would be retried again. This is legitimate
 753   //           in terms of program correctness (even though it is not desired).
 754   __ bne(CCR0, step_four);
 755 
 756   __ bind(done);
 757   __ block_comment("} cmpxchg_oop (shenandoahgc)");
 758 }
 759 
 760 #undef __
 761 
 762 #ifdef COMPILER1
 763 
 764 #define __ ce->masm()->
 765 
 766 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
 767   __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
 768 
 769   ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
 770   __ bind(*stub->entry());
 771 
 772   // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
 773   // This stub is the slowpath of that function.
 774 
 775   assert(stub->pre_val()->is_register(), "pre_val must be a register");
 776   Register pre_val = stub->pre_val()->as_register();
 777 
 778   // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
 779   // ("preloaded mode" of the store barrier).
 780   if (stub->do_load()) {
 781     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false);
 782   }
 783 
 784   // Fast path: Reference is null.
 785   __ cmpdi(CCR0, pre_val, 0);
 786   __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CCR0, Assembler::equal), *stub->continuation());
 787 
 788   // Argument passing via the stack.
 789   __ std(pre_val, -8, R1_SP);
 790 
 791   __ load_const_optimized(R0, bs->pre_barrier_c1_runtime_code_blob()->code_begin());
 792   __ call_stub(R0);
 793 
 794   __ b(*stub->continuation());
 795   __ block_comment("} gen_pre_barrier_stub (shenandoahgc)");
 796 }
 797 
 798 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler *ce,
 799                                                                     ShenandoahLoadReferenceBarrierStub *stub) {
 800   __ block_comment("gen_load_reference_barrier_stub (shenandoahgc) {");
 801 
 802   ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
 803   __ bind(*stub->entry());
 804 
 805   Register obj  = stub->obj()->as_register();
 806   Register res  = stub->result()->as_register();
 807   Register addr = stub->addr()->as_pointer_register();
 808   Register tmp1 = stub->tmp1()->as_register();
 809   Register tmp2 = stub->tmp2()->as_register();
 810   assert_different_registers(addr, res, tmp1, tmp2);
 811 
 812 #ifdef ASSERT
 813   // Ensure that 'res' is 'R3_ARG1' and contains the same value as 'obj' to reduce the number of required
 814   // copy instructions.
 815   assert(R3_RET == res, "res must be r3");
 816   __ cmpd(CCR0, res, obj);
 817   __ asm_assert_eq("result register must contain the reference stored in obj");
 818 #endif
 819 
 820   DecoratorSet decorators = stub->decorators();
 821 
 822   /* ==== Check whether region is in collection set ==== */
 823   // GC status (unstable) has already been verified by 'ShenandoahBarrierSetC1::load_reference_barrier_impl'.
 824   // This stub is the slowpath of that function.
 825 
 826   bool is_strong  = ShenandoahBarrierSet::is_strong_access(decorators);
 827   bool is_weak    = ShenandoahBarrierSet::is_weak_access(decorators);
 828   bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
 829   bool is_native  = ShenandoahBarrierSet::is_native_access(decorators);
 830 
 831   if (is_strong) {
 832     // Check whether object is in collection set.
 833     __ load_const_optimized(tmp2, ShenandoahHeap::in_cset_fast_test_addr(), tmp1);
 834     __ srdi(tmp1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
 835     __ lbzx(tmp2, tmp1, tmp2);
 836 
 837     __ andi_(tmp2, tmp2, 1);
 838     __ bc_far_optimized(Assembler::bcondCRbiIs1_bhintNoHint, __ bi0(CCR0, Assembler::equal), *stub->continuation());
 839   }
 840 
 841   address blob_addr = nullptr;
 842 
 843   if (is_strong) {
 844     if (is_native) {
 845       blob_addr = bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin();
 846     } else {
 847       blob_addr = bs->load_reference_barrier_strong_rt_code_blob()->code_begin();
 848     }
 849   } else if (is_weak) {
 850     blob_addr = bs->load_reference_barrier_weak_rt_code_blob()->code_begin();
 851   } else {
 852     assert(is_phantom, "only remaining strength");
 853     blob_addr = bs->load_reference_barrier_phantom_rt_code_blob()->code_begin();
 854   }
 855 
 856   assert(blob_addr != nullptr, "code blob cannot be found");
 857 
 858   // Argument passing via the stack.  'obj' is passed implicitly (as asserted above).
 859   __ std(addr, -8, R1_SP);
 860 
 861   __ load_const_optimized(tmp1, blob_addr, tmp2);
 862   __ call_stub(tmp1);
 863 
 864   // 'res' is 'R3_RET'.  The result is thus already in the correct register.
 865 
 866   __ b(*stub->continuation());
 867   __ block_comment("} gen_load_reference_barrier_stub (shenandoahgc)");
 868 }
 869 
 870 #undef __
 871 
 872 #define __ sasm->
 873 
 874 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler *sasm) {
 875   __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
 876 
 877   Label runtime, skip_barrier;
 878   BarrierSet *bs = BarrierSet::barrier_set();
 879 
 880   // Argument passing via the stack.
 881   const int caller_stack_slots = 3;
 882 
 883   Register R0_pre_val = R0;
 884   __ ld(R0, -8, R1_SP);
 885   Register R11_tmp1 = R11_scratch1;
 886   __ std(R11_tmp1, -16, R1_SP);
 887   Register R12_tmp2 = R12_scratch2;
 888   __ std(R12_tmp2, -24, R1_SP);
 889 
 890   /* ==== Check whether marking is active ==== */
 891   // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
 892   // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
 893   __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 894 
 895   __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::MARKING);
 896   __ beq(CCR0, skip_barrier);
 897 
 898   /* ==== Add previous value directly to thread-local SATB mark queue ==== */
 899   // Check queue's capacity.  Jump to runtime if no free slot is available.
 900   __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 901   __ cmpdi(CCR0, R12_tmp2, 0);
 902   __ beq(CCR0, runtime);
 903 
 904   // Capacity suffices.  Decrement the queue's size by one slot (size of one oop).
 905   __ addi(R12_tmp2, R12_tmp2, -wordSize);
 906   __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 907 
 908   // Enqueue the previous value and skip the runtime invocation.
 909   __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
 910   __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
 911   __ b(skip_barrier);
 912 
 913   __ bind(runtime);
 914 
 915   /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
 916   // Save to-be-preserved registers.
 917   const int nbytes_save = (MacroAssembler::num_volatile_regs + caller_stack_slots) * BytesPerWord;
 918   __ save_volatile_gprs(R1_SP, -nbytes_save);
 919   __ save_LR_CR(R11_tmp1);
 920   __ push_frame_reg_args(nbytes_save, R11_tmp1);
 921 
 922   // Invoke runtime.
 923   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), R0_pre_val, R16_thread);
 924 
 925   // Restore to-be-preserved registers.
 926   __ pop_frame();
 927   __ restore_LR_CR(R11_tmp1);
 928   __ restore_volatile_gprs(R1_SP, -nbytes_save);
 929 
 930   __ bind(skip_barrier);
 931 
 932   // Restore spilled registers.
 933   __ ld(R11_tmp1, -16, R1_SP);
 934   __ ld(R12_tmp2, -24, R1_SP);
 935 
 936   __ blr();
 937   __ block_comment("} generate_c1_pre_barrier_runtime_stub (shenandoahgc)");
 938 }
 939 
 940 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler *sasm,
 941                                                                                     DecoratorSet decorators) {
 942   __ block_comment("generate_c1_load_reference_barrier_runtime_stub (shenandoahgc) {");
 943 
 944   // Argument passing via the stack.
 945   const int caller_stack_slots = 1;
 946 
 947   // Save to-be-preserved registers.
 948   const int nbytes_save = (MacroAssembler::num_volatile_regs - 1 // 'R3_ARG1' is skipped
 949                            + caller_stack_slots) * BytesPerWord;
 950   __ save_volatile_gprs(R1_SP, -nbytes_save, true, false);
 951 
 952   // Load arguments from stack.
 953   // No load required, as assured by assertions in 'ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub'.
 954   Register R3_obj = R3_ARG1;
 955   Register R4_load_addr = R4_ARG2;
 956   __ ld(R4_load_addr, -8, R1_SP);
 957 
 958   Register R11_tmp = R11_scratch1;
 959 
 960   /* ==== Invoke runtime ==== */
 961   bool is_strong  = ShenandoahBarrierSet::is_strong_access(decorators);
 962   bool is_weak    = ShenandoahBarrierSet::is_weak_access(decorators);
 963   bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
 964   bool is_native  = ShenandoahBarrierSet::is_native_access(decorators);
 965 
 966   address jrt_address = nullptr;
 967 
 968   if (is_strong) {
 969     if (is_native) {
 970       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
 971     } else {
 972       if (UseCompressedOops) {
 973         jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
 974       } else {
 975         jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
 976       }
 977     }
 978   } else if (is_weak) {
 979     assert(!is_native, "weak load reference barrier must not be called off-heap");
 980     if (UseCompressedOops) {
 981       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
 982     } else {
 983       jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
 984     }
 985   } else {
 986     assert(is_phantom, "reference type must be phantom");
 987     assert(is_native, "phantom load reference barrier must be called off-heap");
 988     jrt_address = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
 989   }
 990   assert(jrt_address != nullptr, "load reference barrier runtime routine cannot be found");
 991 
 992   __ save_LR_CR(R11_tmp);
 993   __ push_frame_reg_args(nbytes_save, R11_tmp);
 994 
 995   // Invoke runtime.  Arguments are already stored in the corresponding registers.
 996   __ call_VM_leaf(jrt_address, R3_obj, R4_load_addr);
 997 
 998   // Restore to-be-preserved registers.
 999   __ pop_frame();
1000   __ restore_LR_CR(R11_tmp);
1001   __ restore_volatile_gprs(R1_SP, -nbytes_save, true, false); // Skip 'R3_RET' register.
1002 
1003   __ blr();
1004   __ block_comment("} generate_c1_load_reference_barrier_runtime_stub (shenandoahgc)");
1005 }
1006 
1007 #undef __
1008 
1009 #endif // COMPILER1