src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

*** 29,10 ***
--- 29,11 ---
  #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  #include "gc/shenandoah/shenandoahRuntime.hpp"
  #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+ #include "gc/shenandoah/mode/shenandoahMode.hpp"
  #include "interpreter/interpreter.hpp"
  #include "interpreter/interp_masm.hpp"
  #include "runtime/javaThread.hpp"
  #include "runtime/sharedRuntime.hpp"
  #ifdef COMPILER1

*** 58,11 ***
        Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
        __ ldrb(rscratch1, gc_state);
        if (ShenandoahSATBBarrier && dest_uninitialized) {
          __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
        } else {
!         __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
          __ tst(rscratch1, rscratch2);
          __ br(Assembler::EQ, done);
        }
  
        __ push(saved_regs, sp);
--- 59,11 ---
        Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
        __ ldrb(rscratch1, gc_state);
        if (ShenandoahSATBBarrier && dest_uninitialized) {
          __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
        } else {
!         __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
          __ tst(rscratch1, rscratch2);
          __ br(Assembler::EQ, done);
        }
  
        __ push(saved_regs, sp);

*** 75,10 ***
--- 76,17 ---
        __ bind(done);
      }
    }
  }
  
+ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                                        Register start, Register count, Register tmp, RegSet saved_regs) {
+   if (is_oop) {
+     gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs);
+   }
+ }
+ 
  void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
                                                                   Register obj,
                                                                   Register pre_val,
                                                                   Register thread,
                                                                   Register tmp,

*** 373,10 ***
--- 381,35 ---
      __ pop_call_clobbered_registers();
      __ leave();
    }
  }
  
+ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
+   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
+       return;
+   }
+ 
+   ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
+   CardTable* ct = ctbs->card_table();
+ 
+   __ lsr(obj, obj, CardTable::card_shift());
+ 
+   assert(CardTable::dirty_card_val() == 0, "must be");
+ 
+   __ load_byte_map_base(rscratch1);
+ 
+   if (UseCondCardMark) {
+     Label L_already_dirty;
+     __ ldrb(rscratch2,  Address(obj, rscratch1));
+     __ cbz(rscratch2, L_already_dirty);
+     __ strb(zr, Address(obj, rscratch1));
+     __ bind(L_already_dirty);
+   } else {
+     __ strb(zr, Address(obj, rscratch1));
+   }
+ }
+ 
  void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                               Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
    bool on_oop = is_reference_type(type);
    if (!on_oop) {
      BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);

*** 409,10 ***
--- 442,11 ---
      if (UseCompressedOops) {
        new_val = rscratch2;
        __ mov(new_val, val);
      }
      BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
+     store_check(masm, r3);
    }
  
  }
  
  void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,

*** 593,10 ***
--- 627,39 ---
    } else {
      __ cset(result, Assembler::EQ);
    }
  }
  
+ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+                                                                      Register start, Register count, Register scratch, RegSet saved_regs) {
+   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
+     return;
+   }
+ 
+   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
+   CardTable* ct = bs->card_table();
+ 
+   Label L_loop, L_done;
+   const Register end = count;
+ 
+   __ cbz(count, L_done); // zero count - nothing to do
+ 
+   __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop))); // end = start + count << LogBytesPerHeapOop
+   __ sub(end, end, BytesPerHeapOop); // last element address to make inclusive
+   __ lsr(start, start, CardTable::card_shift());
+   __ lsr(end, end, CardTable::card_shift());
+   __ sub(count, end, start); // number of bytes to copy
+ 
+   __ load_byte_map_base(scratch);
+   __ add(start, start, scratch);
+   __ bind(L_loop);
+   __ strb(zr, Address(start, count));
+   __ subs(count, count, 1);
+   __ br(Assembler::GE, L_loop);
+   __ bind(L_done);
+ }
+ 
  #undef __
  
  #ifdef COMPILER1
  
  #define __ ce->masm()->

*** 693,11 ***
    Label runtime;
  
    // Is marking still active?
    Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
    __ ldrb(tmp, gc_state);
!   __ tbz(tmp, ShenandoahHeap::MARKING_BITPOS, done);
  
    // Can we store original value in the thread's buffer?
    __ ldr(tmp, queue_index);
    __ cbz(tmp, runtime);
  
--- 756,17 ---
    Label runtime;
  
    // Is marking still active?
    Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
    __ ldrb(tmp, gc_state);
!   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
+     __ tbz(tmp, ShenandoahHeap::YOUNG_MARKING_BITPOS, done);
+   } else {
+     __ mov(rscratch2, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
+     __ tst(tmp, rscratch2);
+     __ br(Assembler::EQ, done);
+   }
  
    // Can we store original value in the thread's buffer?
    __ ldr(tmp, queue_index);
    __ cbz(tmp, runtime);