< prev index next >

src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp

Print this page

  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "gc/shared/gcArguments.hpp"
  27 #include "gc/shared/gc_globals.hpp"
  28 #include "macroAssembler_ppc.hpp"
  29 #include "precompiled.hpp"
  30 #include "asm/macroAssembler.inline.hpp"
  31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  32 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  33 #include "gc/shenandoah/shenandoahForwarding.hpp"
  34 #include "gc/shenandoah/shenandoahHeap.hpp"
  35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  37 #include "gc/shenandoah/shenandoahRuntime.hpp"
  38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  39 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"

  40 #include "interpreter/interpreter.hpp"
  41 #include "runtime/javaThread.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "utilities/globalDefinitions.hpp"
  44 #include "vm_version_ppc.hpp"
  45 #ifdef COMPILER1
  46 #include "c1/c1_LIRAssembler.hpp"
  47 #include "c1/c1_MacroAssembler.hpp"
  48 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  49 #endif
  50 
  51 #define __ masm->
  52 
  53 void ShenandoahBarrierSetAssembler::satb_write_barrier(MacroAssembler *masm,
  54                                                        Register base, RegisterOrConstant ind_or_offs,
  55                                                        Register tmp1, Register tmp2, Register tmp3,
  56                                                        MacroAssembler::PreservationLevel preservation_level) {
  57   if (ShenandoahSATBBarrier) {
  58     __ block_comment("satb_write_barrier (shenandoahgc) {");
  59     satb_write_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);

  73     satb_write_barrier_impl(masm, decorators, noreg, noreg, val, tmp1, tmp2, preservation_level);
  74     __ block_comment("} iu_barrier (shenandoahgc)");
  75   }
  76 }
  77 
  78 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
  79                                                            Register base, RegisterOrConstant ind_or_offs,
  80                                                            Register dst,
  81                                                            Register tmp1, Register tmp2,
  82                                                            MacroAssembler::PreservationLevel preservation_level) {
  83   if (ShenandoahLoadRefBarrier) {
  84     __ block_comment("load_reference_barrier (shenandoahgc) {");
  85     load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
  86     __ block_comment("} load_reference_barrier (shenandoahgc)");
  87   }
  88 }
  89 
  90 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
  91                                                        Register src, Register dst, Register count,
  92                                                        Register preserve1, Register preserve2) {
  93   __ block_comment("arraycopy_prologue (shenandoahgc) {");
  94 
  95   Register R11_tmp = R11_scratch1;
  96 
  97   assert_different_registers(src, dst, count, R11_tmp, noreg);
  98   if (preserve1 != noreg) {
  99     // Technically not required, but likely to indicate an error.
 100     assert_different_registers(preserve1, preserve2);
 101   }
 102 
 103   /* ==== Check whether barrier is required (optimizations) ==== */
 104   // Fast path: Component type of array is not a reference type.
 105   if (!is_reference_type(type)) {
 106     return;
 107   }
 108 
 109   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 110 
 111   // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
 112   // any useful information.
 113   if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahIUBarrier && !ShenandoahLoadRefBarrier) {
 114     return;
 115   }
 116 

 117   Label skip_prologue;
 118 
 119   // Fast path: Array is of length zero.
 120   __ cmpdi(CCR0, count, 0);
 121   __ beq(CCR0, skip_prologue);
 122 
 123   /* ==== Check whether barrier is required (gc state) ==== */
 124   __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
 125          R16_thread);
 126 
 127   // The set of garbage collection states requiring barriers depends on the available barrier types and the
 128   // type of the reference in question.
 129   // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
 130   // for the garbage collector.
 131   const int required_states = ShenandoahSATBBarrier && dest_uninitialized
 132                               ? ShenandoahHeap::HAS_FORWARDED
 133                               : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
 134 
 135   __ andi_(R11_tmp, R11_tmp, required_states);
 136   __ beq(CCR0, skip_prologue);
 137 
 138   /* ==== Invoke runtime ==== */
 139   // Save to-be-preserved registers.
 140   int highest_preserve_register_index = 0;
 141   {
 142     if (preserve1 != noreg && preserve1->is_volatile()) {
 143       __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 144     }
 145     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 146       __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 147     }
 148 
 149     __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 150     __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 151     __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 152 
 153     __ save_LR_CR(R11_tmp);

 170   {
 171     __ pop_frame();
 172     __ restore_LR_CR(R11_tmp);
 173 
 174     __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 175     __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 176     __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 177 
 178     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 179       __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 180     }
 181     if (preserve1 != noreg && preserve1->is_volatile()) {
 182       __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 183     }
 184   }
 185 
 186   __ bind(skip_prologue);
 187   __ block_comment("} arraycopy_prologue (shenandoahgc)");
 188 }
 189 










 190 // The to-be-enqueued value can either be determined
 191 // - dynamically by passing the reference's address information (load mode) or
 192 // - statically by passing a register the value is stored in (preloaded mode)
 193 //   - for performance optimizations in cases where the previous value is known (currently not implemented) and
 194 //   - for incremental-update barriers.
 195 //
 196 // decorators:  The previous value's decorator set.
 197 //              In "load mode", the value must equal '0'.
 198 // base:        Base register of the reference's address (load mode).
 199 //              In "preloaded mode", the register must equal 'noreg'.
 200 // ind_or_offs: Index or offset of the reference's address (load mode).
 201 //              If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
 202 // pre_val:     Register holding the to-be-stored value (preloaded mode).
 203 //              In "load mode", this register acts as a temporary register and must
 204 //              thus not be 'noreg'.  In "preloaded mode", its content will be sustained.
 205 // tmp1/tmp2:   Temporary registers, one of which must be non-volatile in "preloaded mode".
 206 void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
 207                                                             Register base, RegisterOrConstant ind_or_offs,
 208                                                             Register pre_val,
 209                                                             Register tmp1, Register tmp2,
 210                                                             MacroAssembler::PreservationLevel preservation_level) {
 211   assert_different_registers(tmp1, tmp2, pre_val, noreg);
 212 
 213   Label skip_barrier;
 214 
 215   /* ==== Determine necessary runtime invocation preservation measures ==== */
 216   const bool needs_frame           = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
 217   const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
 218   const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
 219 
 220   // Check whether marking is active.
 221   __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 222 
 223   __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
 224   __ beq(CCR0, skip_barrier);
 225 
 226   /* ==== Determine the reference's previous value ==== */
 227   bool preloaded_mode = base == noreg;
 228   Register pre_val_save = noreg;
 229 
 230   if (preloaded_mode) {
 231     // Previous value has been passed to the method, so it must not be determined manually.
 232     // In case 'pre_val' is a volatile register, it must be saved across the C-call
 233     // as callers may depend on its value.
 234     // Unless the general purposes registers are saved anyway, one of the temporary registers
 235     // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
 236     if (!preserve_gp_registers && pre_val->is_volatile()) {
 237       pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
 238       assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
 239     }
 240 
 241     if ((decorators & IS_NOT_NULL) != 0) {
 242 #ifdef ASSERT
 243       __ cmpdi(CCR0, pre_val, 0);

 569                            base, ind_or_offs,
 570                            intermediate_dst,
 571                            tmp1, R0,
 572                            preservation_level);
 573 
 574     __ mr_if_needed(dst, intermediate_dst);
 575   } else {
 576     BarrierSetAssembler::load_at(masm, decorators, type,
 577                                  base, ind_or_offs,
 578                                  dst,
 579                                  tmp1, tmp2,
 580                                  preservation_level, L_handle_null);
 581   }
 582 
 583   /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
 584   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
 585     iu_barrier(masm, dst, tmp1, tmp2, preservation_level);
 586   }
 587 }
 588 





















 589 // base:        Base register of the reference's address.
 590 // ind_or_offs: Index or offset of the reference's address.
 591 // val:         To-be-stored value/reference's new value.
 592 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
 593                                              Register base, RegisterOrConstant ind_or_offs, Register val,
 594                                              Register tmp1, Register tmp2, Register tmp3,
 595                                              MacroAssembler::PreservationLevel preservation_level) {
 596   if (is_reference_type(type)) {
 597     if (ShenandoahSATBBarrier) {
 598       satb_write_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
 599     }
 600 
 601     if (ShenandoahIUBarrier && val != noreg) {
 602       iu_barrier(masm, val, tmp1, tmp2, preservation_level, decorators);
 603     }
 604   }
 605 
 606   BarrierSetAssembler::store_at(masm, decorators, type,
 607                                 base, ind_or_offs,
 608                                 val,
 609                                 tmp1, tmp2, tmp3,
 610                                 preservation_level);





 611 }
 612 
 613 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
 614                                                                   Register dst, Register jni_env, Register obj,
 615                                                                   Register tmp, Label &slowpath) {
 616   __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
 617 
 618   assert_different_registers(jni_env, obj, tmp);
 619 
 620   Label done;
 621 
 622   // Fast path: Reference is null (JNI tags are zero for null pointers).
 623   __ cmpdi(CCR0, obj, 0);
 624   __ beq(CCR0, done);
 625 
 626   // Resolve jobject using standard implementation.
 627   BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
 628 
 629   // Check whether heap is stable.
 630   __ lbz(tmp,

 740   // unlikely case should be handled by the smallest possible code.  Instead of emitting a third,
 741   // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
 742   // (passed arguments are identical).
 743   //
 744   // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
 745   // to fail.  Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
 746   // It is thus important to ensure that a re-execution of those steps does not put program correctness
 747   // at risk:
 748   // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
 749   // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
 750   //           fails.  Unless the reference has been updated in the meanwhile once again, this is
 751   //           guaranteed to be the case.
 752   //           In case of a concurrent update, the CAS would be retried again. This is legitimate
 753   //           in terms of program correctness (even though it is not desired).
 754   __ bne(CCR0, step_four);
 755 
 756   __ bind(done);
 757   __ block_comment("} cmpxchg_oop (shenandoahgc)");
 758 }
 759 
































 760 #undef __
 761 
 762 #ifdef COMPILER1
 763 
 764 #define __ ce->masm()->
 765 
 766 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
 767   __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
 768 
 769   ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
 770   __ bind(*stub->entry());
 771 
 772   // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
 773   // This stub is the slowpath of that function.
 774 
 775   assert(stub->pre_val()->is_register(), "pre_val must be a register");
 776   Register pre_val = stub->pre_val()->as_register();
 777 
 778   // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
 779   // ("preloaded mode" of the store barrier).

 875   __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
 876 
 877   Label runtime, skip_barrier;
 878   BarrierSet *bs = BarrierSet::barrier_set();
 879 
 880   // Argument passing via the stack.
 881   const int caller_stack_slots = 3;
 882 
 883   Register R0_pre_val = R0;
 884   __ ld(R0, -8, R1_SP);
 885   Register R11_tmp1 = R11_scratch1;
 886   __ std(R11_tmp1, -16, R1_SP);
 887   Register R12_tmp2 = R12_scratch2;
 888   __ std(R12_tmp2, -24, R1_SP);
 889 
 890   /* ==== Check whether marking is active ==== */
 891   // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
 892   // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
 893   __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 894 
 895   __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::MARKING);
 896   __ beq(CCR0, skip_barrier);
 897 
 898   /* ==== Add previous value directly to thread-local SATB mark queue ==== */
 899   // Check queue's capacity.  Jump to runtime if no free slot is available.
 900   __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 901   __ cmpdi(CCR0, R12_tmp2, 0);
 902   __ beq(CCR0, runtime);
 903 
 904   // Capacity suffices.  Decrement the queue's size by one slot (size of one oop).
 905   __ addi(R12_tmp2, R12_tmp2, -wordSize);
 906   __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 907 
 908   // Enqueue the previous value and skip the runtime invocation.
 909   __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
 910   __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
 911   __ b(skip_barrier);
 912 
 913   __ bind(runtime);
 914 
 915   /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */

  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "gc/shared/gcArguments.hpp"
  27 #include "gc/shared/gc_globals.hpp"
  28 #include "macroAssembler_ppc.hpp"
  29 #include "precompiled.hpp"
  30 #include "asm/macroAssembler.inline.hpp"
  31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  32 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  33 #include "gc/shenandoah/shenandoahForwarding.hpp"
  34 #include "gc/shenandoah/shenandoahHeap.hpp"
  35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  37 #include "gc/shenandoah/shenandoahRuntime.hpp"
  38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  39 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  40 #include "gc/shenandoah/mode/shenandoahMode.hpp"
  41 #include "interpreter/interpreter.hpp"
  42 #include "runtime/javaThread.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "utilities/globalDefinitions.hpp"
  45 #include "vm_version_ppc.hpp"
  46 #ifdef COMPILER1
  47 #include "c1/c1_LIRAssembler.hpp"
  48 #include "c1/c1_MacroAssembler.hpp"
  49 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  50 #endif
  51 
  52 #define __ masm->
  53 
  54 void ShenandoahBarrierSetAssembler::satb_write_barrier(MacroAssembler *masm,
  55                                                        Register base, RegisterOrConstant ind_or_offs,
  56                                                        Register tmp1, Register tmp2, Register tmp3,
  57                                                        MacroAssembler::PreservationLevel preservation_level) {
  58   if (ShenandoahSATBBarrier) {
  59     __ block_comment("satb_write_barrier (shenandoahgc) {");
  60     satb_write_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);

  74     satb_write_barrier_impl(masm, decorators, noreg, noreg, val, tmp1, tmp2, preservation_level);
  75     __ block_comment("} iu_barrier (shenandoahgc)");
  76   }
  77 }
  78 
  79 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
  80                                                            Register base, RegisterOrConstant ind_or_offs,
  81                                                            Register dst,
  82                                                            Register tmp1, Register tmp2,
  83                                                            MacroAssembler::PreservationLevel preservation_level) {
  84   if (ShenandoahLoadRefBarrier) {
  85     __ block_comment("load_reference_barrier (shenandoahgc) {");
  86     load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
  87     __ block_comment("} load_reference_barrier (shenandoahgc)");
  88   }
  89 }
  90 
  91 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
  92                                                        Register src, Register dst, Register count,
  93                                                        Register preserve1, Register preserve2) {


  94   Register R11_tmp = R11_scratch1;
  95 
  96   assert_different_registers(src, dst, count, R11_tmp, noreg);
  97   if (preserve1 != noreg) {
  98     // Technically not required, but likely to indicate an error.
  99     assert_different_registers(preserve1, preserve2);
 100   }
 101 
 102   /* ==== Check whether barrier is required (optimizations) ==== */
 103   // Fast path: Component type of array is not a reference type.
 104   if (!is_reference_type(type)) {
 105     return;
 106   }
 107 
 108   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 109 
 110   // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
 111   // any useful information.
 112   if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahIUBarrier && !ShenandoahLoadRefBarrier) {
 113     return;
 114   }
 115 
 116   __ block_comment("arraycopy_prologue (shenandoahgc) {");
 117   Label skip_prologue;
 118 
 119   // Fast path: Array is of length zero.
 120   __ cmpdi(CCR0, count, 0);
 121   __ beq(CCR0, skip_prologue);
 122 
 123   /* ==== Check whether barrier is required (gc state) ==== */
 124   __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
 125          R16_thread);
 126 
 127   // The set of garbage collection states requiring barriers depends on the available barrier types and the
 128   // type of the reference in question.
 129   // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
 130   // for the garbage collector.
 131   const int required_states = ShenandoahSATBBarrier && dest_uninitialized
 132                               ? ShenandoahHeap::HAS_FORWARDED
 133                               : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING;
 134 
 135   __ andi_(R11_tmp, R11_tmp, required_states);
 136   __ beq(CCR0, skip_prologue);
 137 
 138   /* ==== Invoke runtime ==== */
 139   // Save to-be-preserved registers.
 140   int highest_preserve_register_index = 0;
 141   {
 142     if (preserve1 != noreg && preserve1->is_volatile()) {
 143       __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 144     }
 145     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 146       __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 147     }
 148 
 149     __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 150     __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 151     __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 152 
 153     __ save_LR_CR(R11_tmp);

 170   {
 171     __ pop_frame();
 172     __ restore_LR_CR(R11_tmp);
 173 
 174     __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 175     __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 176     __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 177 
 178     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 179       __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 180     }
 181     if (preserve1 != noreg && preserve1->is_volatile()) {
 182       __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 183     }
 184   }
 185 
 186   __ bind(skip_prologue);
 187   __ block_comment("} arraycopy_prologue (shenandoahgc)");
 188 }
 189 
 190 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 191                                                        Register dst, Register count,
 192                                                        Register preserve) {
 193   if (is_reference_type(type)) {
 194     __ block_comment("arraycopy_epilogue (shenandoahgc) {");
 195     gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
 196     __ block_comment("} arraycopy_epilogue (shenandoahgc)");
 197   }
 198 }
 199 
 200 // The to-be-enqueued value can either be determined
 201 // - dynamically by passing the reference's address information (load mode) or
 202 // - statically by passing a register the value is stored in (preloaded mode)
 203 //   - for performance optimizations in cases where the previous value is known (currently not implemented) and
 204 //   - for incremental-update barriers.
 205 //
 206 // decorators:  The previous value's decorator set.
 207 //              In "load mode", the value must equal '0'.
 208 // base:        Base register of the reference's address (load mode).
 209 //              In "preloaded mode", the register must equal 'noreg'.
 210 // ind_or_offs: Index or offset of the reference's address (load mode).
 211 //              If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
 212 // pre_val:     Register holding the to-be-stored value (preloaded mode).
 213 //              In "load mode", this register acts as a temporary register and must
 214 //              thus not be 'noreg'.  In "preloaded mode", its content will be sustained.
 215 // tmp1/tmp2:   Temporary registers, one of which must be non-volatile in "preloaded mode".
 216 void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
 217                                                             Register base, RegisterOrConstant ind_or_offs,
 218                                                             Register pre_val,
 219                                                             Register tmp1, Register tmp2,
 220                                                             MacroAssembler::PreservationLevel preservation_level) {
 221   assert_different_registers(tmp1, tmp2, pre_val, noreg);
 222 
 223   Label skip_barrier;
 224 
 225   /* ==== Determine necessary runtime invocation preservation measures ==== */
 226   const bool needs_frame           = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
 227   const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
 228   const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
 229 
 230   // Check whether marking is active.
 231   __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 232 
 233   __ andi_(tmp1, tmp1, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
 234   __ beq(CCR0, skip_barrier);
 235 
 236   /* ==== Determine the reference's previous value ==== */
 237   bool preloaded_mode = base == noreg;
 238   Register pre_val_save = noreg;
 239 
 240   if (preloaded_mode) {
 241     // Previous value has been passed to the method, so it must not be determined manually.
 242     // In case 'pre_val' is a volatile register, it must be saved across the C-call
 243     // as callers may depend on its value.
 244     // Unless the general purposes registers are saved anyway, one of the temporary registers
 245     // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
 246     if (!preserve_gp_registers && pre_val->is_volatile()) {
 247       pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
 248       assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
 249     }
 250 
 251     if ((decorators & IS_NOT_NULL) != 0) {
 252 #ifdef ASSERT
 253       __ cmpdi(CCR0, pre_val, 0);

 579                            base, ind_or_offs,
 580                            intermediate_dst,
 581                            tmp1, R0,
 582                            preservation_level);
 583 
 584     __ mr_if_needed(dst, intermediate_dst);
 585   } else {
 586     BarrierSetAssembler::load_at(masm, decorators, type,
 587                                  base, ind_or_offs,
 588                                  dst,
 589                                  tmp1, tmp2,
 590                                  preservation_level, L_handle_null);
 591   }
 592 
 593   /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
 594   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
 595     iu_barrier(masm, dst, tmp1, tmp2, preservation_level);
 596   }
 597 }
 598 
 599 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
 600   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 601       return;
 602   }
 603 
 604   ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
 605   CardTable* ct = ctbs->card_table();
 606   assert_different_registers(base, tmp, R0);
 607 
 608   if (ind_or_offs.is_constant()) {
 609     __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
 610   } else {
 611     __ add(base, ind_or_offs.as_register(), base);
 612   }
 613 
 614   __ load_const_optimized(tmp, (address)ct->byte_map_base(), R0);
 615   __ srdi(base, base, CardTable::card_shift());
 616   __ li(R0, CardTable::dirty_card_val());
 617   __ stbx(R0, tmp, base);
 618 }
 619 
 620 // base:        Base register of the reference's address.
 621 // ind_or_offs: Index or offset of the reference's address.
 622 // val:         To-be-stored value/reference's new value.
 623 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
 624                                              Register base, RegisterOrConstant ind_or_offs, Register val,
 625                                              Register tmp1, Register tmp2, Register tmp3,
 626                                              MacroAssembler::PreservationLevel preservation_level) {
 627   if (is_reference_type(type)) {
 628     if (ShenandoahSATBBarrier) {
 629       satb_write_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
 630     }
 631 
 632     if (ShenandoahIUBarrier && val != noreg) {
 633       iu_barrier(masm, val, tmp1, tmp2, preservation_level, decorators);
 634     }
 635   }
 636 
 637   BarrierSetAssembler::store_at(masm, decorators, type,
 638                                 base, ind_or_offs,
 639                                 val,
 640                                 tmp1, tmp2, tmp3,
 641                                 preservation_level);
 642 
 643   // No need for post barrier if storing NULL
 644   if (is_reference_type(type) && val != noreg) {
 645     store_check(masm, base, ind_or_offs, tmp1);
 646   }
 647 }
 648 
 649 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
 650                                                                   Register dst, Register jni_env, Register obj,
 651                                                                   Register tmp, Label &slowpath) {
 652   __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
 653 
 654   assert_different_registers(jni_env, obj, tmp);
 655 
 656   Label done;
 657 
 658   // Fast path: Reference is null (JNI tags are zero for null pointers).
 659   __ cmpdi(CCR0, obj, 0);
 660   __ beq(CCR0, done);
 661 
 662   // Resolve jobject using standard implementation.
 663   BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
 664 
 665   // Check whether heap is stable.
 666   __ lbz(tmp,

 776   // unlikely case should be handled by the smallest possible code.  Instead of emitting a third,
 777   // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
 778   // (passed arguments are identical).
 779   //
 780   // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
 781   // to fail.  Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
 782   // It is thus important to ensure that a re-execution of those steps does not put program correctness
 783   // at risk:
 784   // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
 785   // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
 786   //           fails.  Unless the reference has been updated in the meanwhile once again, this is
 787   //           guaranteed to be the case.
 788   //           In case of a concurrent update, the CAS would be retried again. This is legitimate
 789   //           in terms of program correctness (even though it is not desired).
 790   __ bne(CCR0, step_four);
 791 
 792   __ bind(done);
 793   __ block_comment("} cmpxchg_oop (shenandoahgc)");
 794 }
 795 
 796 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 797                                                                      Register addr, Register count, Register preserve) {
 798   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 799     return;
 800   }
 801 
 802   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
 803   CardTable* ct = bs->card_table();
 804   assert_different_registers(addr, count, R0);
 805 
 806   Label Lskip_loop, Lstore_loop;
 807 
 808   __ sldi_(count, count, LogBytesPerHeapOop);
 809   __ beq(CCR0, Lskip_loop); // zero length
 810   __ addi(count, count, -BytesPerHeapOop);
 811   __ add(count, addr, count);
 812   // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
 813   __ srdi(addr, addr, CardTable::card_shift());
 814   __ srdi(count, count, CardTable::card_shift());
 815   __ subf(count, addr, count);
 816   __ add_const_optimized(addr, addr, (address)ct->byte_map_base(), R0);
 817   __ addi(count, count, 1);
 818   __ li(R0, 0);
 819   __ mtctr(count);
 820   // Byte store loop
 821   __ bind(Lstore_loop);
 822   __ stb(R0, 0, addr);
 823   __ addi(addr, addr, 1);
 824   __ bdnz(Lstore_loop);
 825   __ bind(Lskip_loop);
 826 }
 827 
 828 #undef __
 829 
 830 #ifdef COMPILER1
 831 
 832 #define __ ce->masm()->
 833 
 834 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
 835   __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
 836 
 837   ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
 838   __ bind(*stub->entry());
 839 
 840   // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
 841   // This stub is the slowpath of that function.
 842 
 843   assert(stub->pre_val()->is_register(), "pre_val must be a register");
 844   Register pre_val = stub->pre_val()->as_register();
 845 
 846   // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
 847   // ("preloaded mode" of the store barrier).

 943   __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
 944 
 945   Label runtime, skip_barrier;
 946   BarrierSet *bs = BarrierSet::barrier_set();
 947 
 948   // Argument passing via the stack.
 949   const int caller_stack_slots = 3;
 950 
 951   Register R0_pre_val = R0;
 952   __ ld(R0, -8, R1_SP);
 953   Register R11_tmp1 = R11_scratch1;
 954   __ std(R11_tmp1, -16, R1_SP);
 955   Register R12_tmp2 = R12_scratch2;
 956   __ std(R12_tmp2, -24, R1_SP);
 957 
 958   /* ==== Check whether marking is active ==== */
 959   // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
 960   // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
 961   __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 962 
 963   __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
 964   __ beq(CCR0, skip_barrier);
 965 
 966   /* ==== Add previous value directly to thread-local SATB mark queue ==== */
 967   // Check queue's capacity.  Jump to runtime if no free slot is available.
 968   __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 969   __ cmpdi(CCR0, R12_tmp2, 0);
 970   __ beq(CCR0, runtime);
 971 
 972   // Capacity suffices.  Decrement the queue's size by one slot (size of one oop).
 973   __ addi(R12_tmp2, R12_tmp2, -wordSize);
 974   __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 975 
 976   // Enqueue the previous value and skip the runtime invocation.
 977   __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
 978   __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
 979   __ b(skip_barrier);
 980 
 981   __ bind(runtime);
 982 
 983   /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
< prev index next >