src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

@@ -1,7 +1,8 @@
  /*
   * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
+  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   *
   * This code is free software; you can redistribute it and/or modify it
   * under the terms of the GNU General Public License version 2 only, as
   * published by the Free Software Foundation.

@@ -29,10 +30,11 @@
  #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  #include "gc/shenandoah/shenandoahRuntime.hpp"
  #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+ #include "gc/shenandoah/mode/shenandoahMode.hpp"
  #include "interpreter/interpreter.hpp"
  #include "interpreter/interp_masm.hpp"
  #include "runtime/javaThread.hpp"
  #include "runtime/sharedRuntime.hpp"
  #ifdef COMPILER1

@@ -45,11 +47,11 @@
  
  void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
                                                         Register src, Register dst, Register count, RegSet saved_regs) {
    if (is_oop) {
      bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
-     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
+     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
  
        Label done;
  
        // Avoid calling runtime if count == 0
        __ cbz(count, done);

@@ -75,10 +77,17 @@
        __ bind(done);
      }
    }
  }
  
+ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                                        Register start, Register count, Register tmp, RegSet saved_regs) {
+   if (ShenandoahCardBarrier && is_oop) {
+     gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs);
+   }
+ }
+ 
  void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
                                                                   Register obj,
                                                                   Register pre_val,
                                                                   Register thread,
                                                                   Register tmp,

@@ -303,18 +312,10 @@
  
    __ bind(heap_stable);
    __ leave();
  }
  
- void ShenandoahBarrierSetAssembler::iu_barrier(MacroAssembler* masm, Register dst, Register tmp) {
-   if (ShenandoahIUBarrier) {
-     __ push_call_clobbered_registers();
-     satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, rscratch1, true, false);
-     __ pop_call_clobbered_registers();
-   }
- }
- 
  //
  // Arguments:
  //
  // Inputs:
  //   src:        oop location to load from, might be clobbered

@@ -373,10 +374,30 @@
      __ pop_call_clobbered_registers();
      __ leave();
    }
  }
  
+ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
+   assert(ShenandoahCardBarrier, "Should have been checked by caller");
+ 
+   __ lsr(obj, obj, CardTable::card_shift());
+ 
+   assert(CardTable::dirty_card_val() == 0, "must be");
+ 
+   __ load_byte_map_base(rscratch1);
+ 
+   if (UseCondCardMark) {
+     Label L_already_dirty;
+     __ ldrb(rscratch2, Address(obj, rscratch1));
+     __ cbz(rscratch2, L_already_dirty);
+     __ strb(zr, Address(obj, rscratch1));
+     __ bind(L_already_dirty);
+   } else {
+     __ strb(zr, Address(obj, rscratch1));
+   }
+ }
+ 
  void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                               Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
    bool on_oop = is_reference_type(type);
    if (!on_oop) {
      BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);

@@ -401,18 +422,20 @@
                                 false /* expand_call */);
  
    if (val == noreg) {
      BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
    } else {
-     iu_barrier(masm, val, tmp1);
-     // G1 barrier needs uncompressed oop for region cross check.
+     // Barrier needs uncompressed oop for region cross check.
      Register new_val = val;
      if (UseCompressedOops) {
        new_val = rscratch2;
        __ mov(new_val, val);
      }
      BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
+     if (ShenandoahCardBarrier) {
+       store_check(masm, tmp3);
+     }
    }
  
  }
  
  void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,

@@ -593,10 +616,39 @@
    } else {
      __ cset(result, Assembler::EQ);
    }
  }
  
+ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+                                                                      Register start, Register count, Register scratch, RegSet saved_regs) {
+   assert(ShenandoahCardBarrier, "Should have been checked by caller");
+ 
+   Label L_loop, L_done;
+   const Register end = count;
+ 
+   // Zero count? Nothing to do.
+   __ cbz(count, L_done);
+ 
+   // end = start + count << LogBytesPerHeapOop
+   // last element address to make inclusive
+   __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
+   __ sub(end, end, BytesPerHeapOop);
+   __ lsr(start, start, CardTable::card_shift());
+   __ lsr(end, end, CardTable::card_shift());
+ 
+   // number of bytes to copy
+   __ sub(count, end, start);
+ 
+   __ load_byte_map_base(scratch);
+   __ add(start, start, scratch);
+   __ bind(L_loop);
+   __ strb(zr, Address(start, count));
+   __ subs(count, count, 1);
+   __ br(Assembler::GE, L_loop);
+   __ bind(L_done);
+ }
+ 
  #undef __
  
  #ifdef COMPILER1
  
  #define __ ce->masm()->