< prev index next >

src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp

Print this page

  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "gc/shared/gcArguments.hpp"
  27 #include "gc/shared/gc_globals.hpp"
  28 #include "macroAssembler_ppc.hpp"
  29 #include "precompiled.hpp"
  30 #include "asm/macroAssembler.inline.hpp"
  31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  32 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  33 #include "gc/shenandoah/shenandoahForwarding.hpp"
  34 #include "gc/shenandoah/shenandoahHeap.hpp"
  35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  37 #include "gc/shenandoah/shenandoahRuntime.hpp"
  38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  39 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"

  40 #include "interpreter/interpreter.hpp"
  41 #include "runtime/sharedRuntime.hpp"
  42 #include "runtime/thread.hpp"
  43 #include "utilities/globalDefinitions.hpp"
  44 #include "vm_version_ppc.hpp"
  45 
  46 #ifdef COMPILER1
  47 
  48 #include "c1/c1_LIRAssembler.hpp"
  49 #include "c1/c1_MacroAssembler.hpp"
  50 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  51 
  52 #endif
  53 
  54 #define __ masm->
  55 
  56 void ShenandoahBarrierSetAssembler::satb_write_barrier(MacroAssembler *masm,
  57                                                        Register base, RegisterOrConstant ind_or_offs,
  58                                                        Register tmp1, Register tmp2, Register tmp3,
  59                                                        MacroAssembler::PreservationLevel preservation_level) {

  76     satb_write_barrier_impl(masm, decorators, noreg, noreg, val, tmp1, tmp2, preservation_level);
  77     __ block_comment("} iu_barrier (shenandoahgc)");
  78   }
  79 }
  80 
  81 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
  82                                                            Register base, RegisterOrConstant ind_or_offs,
  83                                                            Register dst,
  84                                                            Register tmp1, Register tmp2,
  85                                                            MacroAssembler::PreservationLevel preservation_level) {
  86   if (ShenandoahLoadRefBarrier) {
  87     __ block_comment("load_reference_barrier (shenandoahgc) {");
  88     load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
  89     __ block_comment("} load_reference_barrier (shenandoahgc)");
  90   }
  91 }
  92 
  93 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
  94                                                        Register src, Register dst, Register count,
  95                                                        Register preserve1, Register preserve2) {
  96   __ block_comment("arraycopy_prologue (shenandoahgc) {");
  97 
  98   Register R11_tmp = R11_scratch1;
  99 
 100   assert_different_registers(src, dst, count, R11_tmp, noreg);
 101   if (preserve1 != noreg) {
 102     // Technically not required, but likely to indicate an error.
 103     assert_different_registers(preserve1, preserve2);
 104   }
 105 
 106   /* ==== Check whether barrier is required (optimizations) ==== */
 107   // Fast path: Component type of array is not a reference type.
 108   if (!is_reference_type(type)) {
 109     return;
 110   }
 111 
 112   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 113 
 114   // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
 115   // any useful information.
 116   if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahIUBarrier && !ShenandoahLoadRefBarrier) {
 117     return;
 118   }
 119 

 120   Label skip_prologue;
 121 
 122   // Fast path: Array is of length zero.
 123   __ cmpdi(CCR0, count, 0);
 124   __ beq(CCR0, skip_prologue);
 125 
 126   /* ==== Check whether barrier is required (gc state) ==== */
 127   __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
 128          R16_thread);
 129 
 130   // The set of garbage collection states requiring barriers depends on the available barrier types and the
 131   // type of the reference in question.
 132   // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
 133   // for the garbage collector.
 134   const int required_states = ShenandoahSATBBarrier && dest_uninitialized
 135                               ? ShenandoahHeap::HAS_FORWARDED
 136                               : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
 137 
 138   __ andi_(R11_tmp, R11_tmp, required_states);
 139   __ beq(CCR0, skip_prologue);
 140 
 141   /* ==== Invoke runtime ==== */
 142   // Save to-be-preserved registers.
 143   int highest_preserve_register_index = 0;
 144   {
 145     if (preserve1 != noreg && preserve1->is_volatile()) {
 146       __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 147     }
 148     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 149       __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 150     }
 151 
 152     __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 153     __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 154     __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 155 
 156     __ save_LR_CR(R11_tmp);

 173   {
 174     __ pop_frame();
 175     __ restore_LR_CR(R11_tmp);
 176 
 177     __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 178     __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 179     __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 180 
 181     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 182       __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 183     }
 184     if (preserve1 != noreg && preserve1->is_volatile()) {
 185       __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 186     }
 187   }
 188 
 189   __ bind(skip_prologue);
 190   __ block_comment("} arraycopy_prologue (shenandoahgc)");
 191 }
 192 










 193 // The to-be-enqueued value can either be determined
 194 // - dynamically by passing the reference's address information (load mode) or
 195 // - statically by passing a register the value is stored in (preloaded mode)
 196 //   - for performance optimizations in cases where the previous value is known (currently not implemented) and
 197 //   - for incremental-update barriers.
 198 //
 199 // decorators:  The previous value's decorator set.
 200 //              In "load mode", the value must equal '0'.
 201 // base:        Base register of the reference's address (load mode).
 202 //              In "preloaded mode", the register must equal 'noreg'.
 203 // ind_or_offs: Index or offset of the reference's address (load mode).
 204 //              If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
 205 // pre_val:     Register holding the to-be-stored value (preloaded mode).
 206 //              In "load mode", this register acts as a temporary register and must
 207 //              thus not be 'noreg'.  In "preloaded mode", its content will be sustained.
 208 // tmp1/tmp2:   Temporary registers, one of which must be non-volatile in "preloaded mode".
 209 void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
 210                                                             Register base, RegisterOrConstant ind_or_offs,
 211                                                             Register pre_val,
 212                                                             Register tmp1, Register tmp2,
 213                                                             MacroAssembler::PreservationLevel preservation_level) {
 214   assert_different_registers(tmp1, tmp2, pre_val, noreg);
 215 
 216   Label skip_barrier;
 217 
 218   /* ==== Determine necessary runtime invocation preservation measures ==== */
 219   const bool needs_frame           = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
 220   const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
 221   const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
 222 
 223   // Check whether marking is active.
 224   __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 225 
 226   __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING);
 227   __ beq(CCR0, skip_barrier);
 228 
 229   /* ==== Determine the reference's previous value ==== */
 230   bool preloaded_mode = base == noreg;
 231   Register pre_val_save = noreg;
 232 
 233   if (preloaded_mode) {
 234     // Previous value has been passed to the method, so it must not be determined manually.
 235     // In case 'pre_val' is a volatile register, it must be saved across the C-call
 236     // as callers may depend on its value.
 237     // Unless the general purposes registers are saved anyway, one of the temporary registers
 238     // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
 239     if (!preserve_gp_registers && pre_val->is_volatile()) {
 240       pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
 241       assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
 242     }
 243 
 244     if ((decorators & IS_NOT_NULL) != 0) {
 245 #ifdef ASSERT
 246       __ cmpdi(CCR0, pre_val, 0);

 572                            base, ind_or_offs,
 573                            intermediate_dst,
 574                            tmp1, R0,
 575                            preservation_level);
 576 
 577     __ mr_if_needed(dst, intermediate_dst);
 578   } else {
 579     BarrierSetAssembler::load_at(masm, decorators, type,
 580                                  base, ind_or_offs,
 581                                  dst,
 582                                  tmp1, tmp2,
 583                                  preservation_level, L_handle_null);
 584   }
 585 
 586   /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
 587   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
 588     iu_barrier(masm, dst, tmp1, tmp2, preservation_level);
 589   }
 590 }
 591 





















 592 // base:        Base register of the reference's address.
 593 // ind_or_offs: Index or offset of the reference's address.
 594 // val:         To-be-stored value/reference's new value.
 595 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
 596                                              Register base, RegisterOrConstant ind_or_offs, Register val,
 597                                              Register tmp1, Register tmp2, Register tmp3,
 598                                              MacroAssembler::PreservationLevel preservation_level) {
 599   if (is_reference_type(type)) {
 600     if (ShenandoahSATBBarrier) {
 601       satb_write_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
 602     }
 603 
 604     if (ShenandoahIUBarrier && val != noreg) {
 605       iu_barrier(masm, val, tmp1, tmp2, preservation_level, decorators);
 606     }
 607   }
 608 
 609   BarrierSetAssembler::store_at(masm, decorators, type,
 610                                 base, ind_or_offs,
 611                                 val,
 612                                 tmp1, tmp2, tmp3,
 613                                 preservation_level);





 614 }
 615 
 616 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
 617                                                                   Register dst, Register jni_env, Register obj,
 618                                                                   Register tmp, Label &slowpath) {
 619   __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
 620 
 621   assert_different_registers(jni_env, obj, tmp);
 622 
 623   Label done;
 624 
 625   // Fast path: Reference is null (JNI tags are zero for null pointers).
 626   __ cmpdi(CCR0, obj, 0);
 627   __ beq(CCR0, done);
 628 
 629   // Resolve jobject using standard implementation.
 630   BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
 631 
 632   // Check whether heap is stable.
 633   __ lbz(tmp,

 743   // unlikely case should be handled by the smallest possible code.  Instead of emitting a third,
 744   // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
 745   // (passed arguments are identical).
 746   //
 747   // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
 748   // to fail.  Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
 749   // It is thus important to ensure that a re-execution of those steps does not put program correctness
 750   // at risk:
 751   // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
 752   // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
 753   //           fails.  Unless the reference has been updated in the meanwhile once again, this is
 754   //           guaranteed to be the case.
 755   //           In case of a concurrent update, the CAS would be retried again. This is legitimate
 756   //           in terms of program correctness (even though it is not desired).
 757   __ bne(CCR0, step_four);
 758 
 759   __ bind(done);
 760   __ block_comment("} cmpxchg_oop (shenandoahgc)");
 761 }
 762 
































 763 #undef __
 764 
 765 #ifdef COMPILER1
 766 
 767 #define __ ce->masm()->
 768 
 769 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
 770   __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
 771 
 772   ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
 773   __ bind(*stub->entry());
 774 
 775   // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
 776   // This stub is the slowpath of that function.
 777 
 778   assert(stub->pre_val()->is_register(), "pre_val must be a register");
 779   Register pre_val = stub->pre_val()->as_register();
 780 
 781   // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
 782   // ("preloaded mode" of the store barrier).

 878   __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
 879 
 880   Label runtime, skip_barrier;
 881   BarrierSet *bs = BarrierSet::barrier_set();
 882 
 883   // Argument passing via the stack.
 884   const int caller_stack_slots = 3;
 885 
 886   Register R0_pre_val = R0;
 887   __ ld(R0, -8, R1_SP);
 888   Register R11_tmp1 = R11_scratch1;
 889   __ std(R11_tmp1, -16, R1_SP);
 890   Register R12_tmp2 = R12_scratch2;
 891   __ std(R12_tmp2, -24, R1_SP);
 892 
 893   /* ==== Check whether marking is active ==== */
 894   // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
 895   // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
 896   __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 897 
 898   __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::MARKING);
 899   __ beq(CCR0, skip_barrier);
 900 
 901   /* ==== Add previous value directly to thread-local SATB mark queue ==== */
 902   // Check queue's capacity.  Jump to runtime if no free slot is available.
 903   __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 904   __ cmpdi(CCR0, R12_tmp2, 0);
 905   __ beq(CCR0, runtime);
 906 
 907   // Capacity suffices.  Decrement the queue's size by one slot (size of one oop).
 908   __ addi(R12_tmp2, R12_tmp2, -wordSize);
 909   __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 910 
 911   // Enqueue the previous value and skip the runtime invocation.
 912   __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
 913   __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
 914   __ b(skip_barrier);
 915 
 916   __ bind(runtime);
 917 
 918   /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */

  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "gc/shared/gcArguments.hpp"
  27 #include "gc/shared/gc_globals.hpp"
  28 #include "macroAssembler_ppc.hpp"
  29 #include "precompiled.hpp"
  30 #include "asm/macroAssembler.inline.hpp"
  31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  32 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  33 #include "gc/shenandoah/shenandoahForwarding.hpp"
  34 #include "gc/shenandoah/shenandoahHeap.hpp"
  35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  37 #include "gc/shenandoah/shenandoahRuntime.hpp"
  38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  39 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  40 #include "gc/shenandoah/mode/shenandoahMode.hpp"
  41 #include "interpreter/interpreter.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "runtime/thread.hpp"
  44 #include "utilities/globalDefinitions.hpp"
  45 #include "vm_version_ppc.hpp"
  46 
  47 #ifdef COMPILER1
  48 
  49 #include "c1/c1_LIRAssembler.hpp"
  50 #include "c1/c1_MacroAssembler.hpp"
  51 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  52 
  53 #endif
  54 
  55 #define __ masm->
  56 
  57 void ShenandoahBarrierSetAssembler::satb_write_barrier(MacroAssembler *masm,
  58                                                        Register base, RegisterOrConstant ind_or_offs,
  59                                                        Register tmp1, Register tmp2, Register tmp3,
  60                                                        MacroAssembler::PreservationLevel preservation_level) {

  77     satb_write_barrier_impl(masm, decorators, noreg, noreg, val, tmp1, tmp2, preservation_level);
  78     __ block_comment("} iu_barrier (shenandoahgc)");
  79   }
  80 }
  81 
  82 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
  83                                                            Register base, RegisterOrConstant ind_or_offs,
  84                                                            Register dst,
  85                                                            Register tmp1, Register tmp2,
  86                                                            MacroAssembler::PreservationLevel preservation_level) {
  87   if (ShenandoahLoadRefBarrier) {
  88     __ block_comment("load_reference_barrier (shenandoahgc) {");
  89     load_reference_barrier_impl(masm, decorators, base, ind_or_offs, dst, tmp1, tmp2, preservation_level);
  90     __ block_comment("} load_reference_barrier (shenandoahgc)");
  91   }
  92 }
  93 
  94 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
  95                                                        Register src, Register dst, Register count,
  96                                                        Register preserve1, Register preserve2) {


  97   Register R11_tmp = R11_scratch1;
  98 
  99   assert_different_registers(src, dst, count, R11_tmp, noreg);
 100   if (preserve1 != noreg) {
 101     // Technically not required, but likely to indicate an error.
 102     assert_different_registers(preserve1, preserve2);
 103   }
 104 
 105   /* ==== Check whether barrier is required (optimizations) ==== */
 106   // Fast path: Component type of array is not a reference type.
 107   if (!is_reference_type(type)) {
 108     return;
 109   }
 110 
 111   bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 112 
 113   // Fast path: No barrier required if for every barrier type, it is either disabled or would not store
 114   // any useful information.
 115   if ((!ShenandoahSATBBarrier || dest_uninitialized) && !ShenandoahIUBarrier && !ShenandoahLoadRefBarrier) {
 116     return;
 117   }
 118 
 119   __ block_comment("arraycopy_prologue (shenandoahgc) {");
 120   Label skip_prologue;
 121 
 122   // Fast path: Array is of length zero.
 123   __ cmpdi(CCR0, count, 0);
 124   __ beq(CCR0, skip_prologue);
 125 
 126   /* ==== Check whether barrier is required (gc state) ==== */
 127   __ lbz(R11_tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
 128          R16_thread);
 129 
 130   // The set of garbage collection states requiring barriers depends on the available barrier types and the
 131   // type of the reference in question.
 132   // For instance, satb barriers may be skipped if it is certain that the overridden values are not relevant
 133   // for the garbage collector.
 134   const int required_states = ShenandoahSATBBarrier && dest_uninitialized
 135                               ? ShenandoahHeap::HAS_FORWARDED
 136                               : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING;
 137 
 138   __ andi_(R11_tmp, R11_tmp, required_states);
 139   __ beq(CCR0, skip_prologue);
 140 
 141   /* ==== Invoke runtime ==== */
 142   // Save to-be-preserved registers.
 143   int highest_preserve_register_index = 0;
 144   {
 145     if (preserve1 != noreg && preserve1->is_volatile()) {
 146       __ std(preserve1, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 147     }
 148     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 149       __ std(preserve2, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 150     }
 151 
 152     __ std(src, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 153     __ std(dst, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 154     __ std(count, -BytesPerWord * ++highest_preserve_register_index, R1_SP);
 155 
 156     __ save_LR_CR(R11_tmp);

 173   {
 174     __ pop_frame();
 175     __ restore_LR_CR(R11_tmp);
 176 
 177     __ ld(count, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 178     __ ld(dst, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 179     __ ld(src, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 180 
 181     if (preserve2 != noreg && preserve2 != preserve1 && preserve2->is_volatile()) {
 182       __ ld(preserve2, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 183     }
 184     if (preserve1 != noreg && preserve1->is_volatile()) {
 185       __ ld(preserve1, -BytesPerWord * highest_preserve_register_index--, R1_SP);
 186     }
 187   }
 188 
 189   __ bind(skip_prologue);
 190   __ block_comment("} arraycopy_prologue (shenandoahgc)");
 191 }
 192 
 193 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 194                                                        Register dst, Register count,
 195                                                        Register preserve) {
 196   if (is_reference_type(type)) {
 197     __ block_comment("arraycopy_epilogue (shenandoahgc) {");
 198     gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
 199     __ block_comment("} arraycopy_epilogue (shenandoahgc)");
 200   }
 201 }
 202 
 203 // The to-be-enqueued value can either be determined
 204 // - dynamically by passing the reference's address information (load mode) or
 205 // - statically by passing a register the value is stored in (preloaded mode)
 206 //   - for performance optimizations in cases where the previous value is known (currently not implemented) and
 207 //   - for incremental-update barriers.
 208 //
 209 // decorators:  The previous value's decorator set.
 210 //              In "load mode", the value must equal '0'.
 211 // base:        Base register of the reference's address (load mode).
 212 //              In "preloaded mode", the register must equal 'noreg'.
 213 // ind_or_offs: Index or offset of the reference's address (load mode).
 214 //              If 'base' equals 'noreg' (preloaded mode), the passed value is ignored.
 215 // pre_val:     Register holding the to-be-stored value (preloaded mode).
 216 //              In "load mode", this register acts as a temporary register and must
 217 //              thus not be 'noreg'.  In "preloaded mode", its content will be sustained.
 218 // tmp1/tmp2:   Temporary registers, one of which must be non-volatile in "preloaded mode".
 219 void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
 220                                                             Register base, RegisterOrConstant ind_or_offs,
 221                                                             Register pre_val,
 222                                                             Register tmp1, Register tmp2,
 223                                                             MacroAssembler::PreservationLevel preservation_level) {
 224   assert_different_registers(tmp1, tmp2, pre_val, noreg);
 225 
 226   Label skip_barrier;
 227 
 228   /* ==== Determine necessary runtime invocation preservation measures ==== */
 229   const bool needs_frame           = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
 230   const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
 231   const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
 232 
 233   // Check whether marking is active.
 234   __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 235 
 236   __ andi_(tmp1, tmp1, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
 237   __ beq(CCR0, skip_barrier);
 238 
 239   /* ==== Determine the reference's previous value ==== */
 240   bool preloaded_mode = base == noreg;
 241   Register pre_val_save = noreg;
 242 
 243   if (preloaded_mode) {
 244     // Previous value has been passed to the method, so it must not be determined manually.
 245     // In case 'pre_val' is a volatile register, it must be saved across the C-call
 246     // as callers may depend on its value.
 247     // Unless the general purposes registers are saved anyway, one of the temporary registers
 248     // (i.e., 'tmp1' and 'tmp2') is used to the preserve 'pre_val'.
 249     if (!preserve_gp_registers && pre_val->is_volatile()) {
 250       pre_val_save = !tmp1->is_volatile() ? tmp1 : tmp2;
 251       assert(!pre_val_save->is_volatile(), "at least one of the temporary registers must be non-volatile");
 252     }
 253 
 254     if ((decorators & IS_NOT_NULL) != 0) {
 255 #ifdef ASSERT
 256       __ cmpdi(CCR0, pre_val, 0);

 582                            base, ind_or_offs,
 583                            intermediate_dst,
 584                            tmp1, R0,
 585                            preservation_level);
 586 
 587     __ mr_if_needed(dst, intermediate_dst);
 588   } else {
 589     BarrierSetAssembler::load_at(masm, decorators, type,
 590                                  base, ind_or_offs,
 591                                  dst,
 592                                  tmp1, tmp2,
 593                                  preservation_level, L_handle_null);
 594   }
 595 
 596   /* ==== Apply keep-alive barrier, if required (e.g., to inhibit weak reference resurrection) ==== */
 597   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
 598     iu_barrier(masm, dst, tmp1, tmp2, preservation_level);
 599   }
 600 }
 601 
 602 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
 603   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 604       return;
 605   }
 606 
 607   ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
 608   CardTable* ct = ctbs->card_table();
 609   assert_different_registers(base, tmp, R0);
 610 
 611   if (ind_or_offs.is_constant()) {
 612     __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
 613   } else {
 614     __ add(base, ind_or_offs.as_register(), base);
 615   }
 616 
 617   __ load_const_optimized(tmp, (address)ct->byte_map_base(), R0);
 618   __ srdi(base, base, CardTable::card_shift());
 619   __ li(R0, CardTable::dirty_card_val());
 620   __ stbx(R0, tmp, base);
 621 }
 622 
 623 // base:        Base register of the reference's address.
 624 // ind_or_offs: Index or offset of the reference's address.
 625 // val:         To-be-stored value/reference's new value.
 626 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
 627                                              Register base, RegisterOrConstant ind_or_offs, Register val,
 628                                              Register tmp1, Register tmp2, Register tmp3,
 629                                              MacroAssembler::PreservationLevel preservation_level) {
 630   if (is_reference_type(type)) {
 631     if (ShenandoahSATBBarrier) {
 632       satb_write_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
 633     }
 634 
 635     if (ShenandoahIUBarrier && val != noreg) {
 636       iu_barrier(masm, val, tmp1, tmp2, preservation_level, decorators);
 637     }
 638   }
 639 
 640   BarrierSetAssembler::store_at(masm, decorators, type,
 641                                 base, ind_or_offs,
 642                                 val,
 643                                 tmp1, tmp2, tmp3,
 644                                 preservation_level);
 645 
 646   // No need for post barrier if storing NULL
 647   if (is_reference_type(type) && val != noreg) {
 648     store_check(masm, base, ind_or_offs, tmp1);
 649   }
 650 }
 651 
 652 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
 653                                                                   Register dst, Register jni_env, Register obj,
 654                                                                   Register tmp, Label &slowpath) {
 655   __ block_comment("try_resolve_jobject_in_native (shenandoahgc) {");
 656 
 657   assert_different_registers(jni_env, obj, tmp);
 658 
 659   Label done;
 660 
 661   // Fast path: Reference is null (JNI tags are zero for null pointers).
 662   __ cmpdi(CCR0, obj, 0);
 663   __ beq(CCR0, done);
 664 
 665   // Resolve jobject using standard implementation.
 666   BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
 667 
 668   // Check whether heap is stable.
 669   __ lbz(tmp,

 779   // unlikely case should be handled by the smallest possible code.  Instead of emitting a third,
 780   // explicit CAS operation, the code jumps back and reuses the first CAS operation (step 1)
 781   // (passed arguments are identical).
 782   //
 783   // A failure of the CAS operation in step 1 would imply that the overall CAS operation is supposed
 784   // to fail.  Jumping back to step 1 requires, however, that step 2 and step 3 are re-executed as well.
 785   // It is thus important to ensure that a re-execution of those steps does not put program correctness
 786   // at risk:
 787   // - Step 2: Either terminates in failure (desired result) or falls through to step 3.
 788   // - Step 3: Terminates if the comparison between the forwarded, fetched pointer and the expected value
 789   //           fails.  Unless the reference has been updated in the meanwhile once again, this is
 790   //           guaranteed to be the case.
 791   //           In case of a concurrent update, the CAS would be retried again. This is legitimate
 792   //           in terms of program correctness (even though it is not desired).
 793   __ bne(CCR0, step_four);
 794 
 795   __ bind(done);
 796   __ block_comment("} cmpxchg_oop (shenandoahgc)");
 797 }
 798 
 799 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 800                                                                      Register addr, Register count, Register preserve) {
 801   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 802     return;
 803   }
 804 
 805   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
 806   CardTable* ct = bs->card_table();
 807   assert_different_registers(addr, count, R0);
 808 
 809   Label Lskip_loop, Lstore_loop;
 810 
 811   __ sldi_(count, count, LogBytesPerHeapOop);
 812   __ beq(CCR0, Lskip_loop); // zero length
 813   __ addi(count, count, -BytesPerHeapOop);
 814   __ add(count, addr, count);
 815   // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
 816   __ srdi(addr, addr, CardTable::card_shift());
 817   __ srdi(count, count, CardTable::card_shift());
 818   __ subf(count, addr, count);
 819   __ add_const_optimized(addr, addr, (address)ct->byte_map_base(), R0);
 820   __ addi(count, count, 1);
 821   __ li(R0, 0);
 822   __ mtctr(count);
 823   // Byte store loop
 824   __ bind(Lstore_loop);
 825   __ stb(R0, 0, addr);
 826   __ addi(addr, addr, 1);
 827   __ bdnz(Lstore_loop);
 828   __ bind(Lskip_loop);
 829 }
 830 
 831 #undef __
 832 
 833 #ifdef COMPILER1
 834 
 835 #define __ ce->masm()->
 836 
 837 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler *ce, ShenandoahPreBarrierStub *stub) {
 838   __ block_comment("gen_pre_barrier_stub (shenandoahgc) {");
 839 
 840   ShenandoahBarrierSetC1 *bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
 841   __ bind(*stub->entry());
 842 
 843   // GC status has already been verified by 'ShenandoahBarrierSetC1::pre_barrier'.
 844   // This stub is the slowpath of that function.
 845 
 846   assert(stub->pre_val()->is_register(), "pre_val must be a register");
 847   Register pre_val = stub->pre_val()->as_register();
 848 
 849   // If 'do_load()' returns false, the to-be-stored value is already available in 'stub->pre_val()'
 850   // ("preloaded mode" of the store barrier).

 946   __ block_comment("generate_c1_pre_barrier_runtime_stub (shenandoahgc) {");
 947 
 948   Label runtime, skip_barrier;
 949   BarrierSet *bs = BarrierSet::barrier_set();
 950 
 951   // Argument passing via the stack.
 952   const int caller_stack_slots = 3;
 953 
 954   Register R0_pre_val = R0;
 955   __ ld(R0, -8, R1_SP);
 956   Register R11_tmp1 = R11_scratch1;
 957   __ std(R11_tmp1, -16, R1_SP);
 958   Register R12_tmp2 = R12_scratch2;
 959   __ std(R12_tmp2, -24, R1_SP);
 960 
 961   /* ==== Check whether marking is active ==== */
 962   // Even though gc status was checked in 'ShenandoahBarrierSetAssembler::gen_pre_barrier_stub',
 963   // another check is required as a safepoint might have been reached in the meantime (JDK-8140588).
 964   __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
 965 
 966   __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
 967   __ beq(CCR0, skip_barrier);
 968 
 969   /* ==== Add previous value directly to thread-local SATB mark queue ==== */
 970   // Check queue's capacity.  Jump to runtime if no free slot is available.
 971   __ ld(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 972   __ cmpdi(CCR0, R12_tmp2, 0);
 973   __ beq(CCR0, runtime);
 974 
 975   // Capacity suffices.  Decrement the queue's size by one slot (size of one oop).
 976   __ addi(R12_tmp2, R12_tmp2, -wordSize);
 977   __ std(R12_tmp2, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
 978 
 979   // Enqueue the previous value and skip the runtime invocation.
 980   __ ld(R11_tmp1, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
 981   __ stdx(R0_pre_val, R11_tmp1, R12_tmp2);
 982   __ b(skip_barrier);
 983 
 984   __ bind(runtime);
 985 
 986   /* ==== Invoke runtime to commit SATB mark queue to gc and allocate a new buffer ==== */
< prev index next >