< prev index next >

src/hotspot/cpu/aarch64/aarch64.ad

Print this page

        

@@ -1,7 +1,7 @@
 //
-// Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
 // Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
 // under the terms of the GNU General Public License version 2 only, as

@@ -998,10 +998,15 @@
 #include "asm/macroAssembler.hpp"
 #include "gc/shared/cardTable.hpp"
 #include "gc/shared/cardTableBarrierSet.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "opto/addnode.hpp"
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#endif
 
 class CallStubImpl {
 
   //--------------------------------------------------------------
   //---<  Used for optimization in Compile::shorten_branches  >---

@@ -1848,10 +1853,16 @@
   // MemBarAcquire pair. There may be an extra If test introduced in
   // the CAS case, when the boolean result of the CAS is tested by the
   // caller. In that case an extra Region and AliasIdxBot Phi may be
   // introduced before the MergeMem
   //
+  // Shenandoah includes a pre barrier between the leading membar and
+  // the volatile object store/cas but its effect on the memory
+  // subgraph between the leading and trailing barriers doesn't
+  // require special predicates: the predicates for the default case
+  // work unmodified.
+  //
   // So, the upshot is that in all cases the subgraph will include a
   // *normal* memory subgraph betwen the leading membar and its child
   // membar: either a normal volatile put graph including a releasing
   // StoreX and terminating with a trailing volatile membar or card
   // mark volatile membar; or a normal CAS graph including a

@@ -1982,19 +1993,31 @@
 
     Node *x = NULL;
     StoreNode * st = NULL;
     LoadStoreNode *cas = NULL;
     MergeMemNode *mm = NULL;
+    MergeMemNode *mm2 = NULL;
 
     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
       x = mem->fast_out(i);
       if (x->is_MergeMem()) {
-        if (mm != NULL) {
-          return NULL;
-        }
-        // two merge mems is one too many
-        mm = x->as_MergeMem();
+        if (UseShenandoahGC) {
+          // three merge mems is one too many for Shenandoah
+          if (mm == NULL) {
+            mm = x->as_MergeMem();
+          } else if (mm2 == NULL) {
+            mm2 = x->as_MergeMem();
+          } else {
+            return NULL;
+          }
+        } else {
+          // two merge mems is one too many
+          if (mm != NULL) {
+            return NULL;
+          }
+          mm = x->as_MergeMem();
+        }
       } else if (x->is_Store() && x->as_Store()->is_release() && x->Opcode() != Op_StoreCM) {
         // two releasing stores/CAS nodes is one too many
         if (st != NULL || cas != NULL) {
           return NULL;
         }

@@ -2010,12 +2033,13 @@
     // must have a store or a cas
     if (!st && !cas) {
       return NULL;
     }
 
-    // must have a merge
-    if (!mm) {
+    // must have a merge if we also have st
+    // can only have a second merge if UseShenandoahGC
+    if (st && (!mm || (!UseShenandoahGC && mm2))) {
       return NULL;
     }
 
     Node *feed = NULL;
     if (cas) {

@@ -2034,16 +2058,27 @@
       feed = st;
     }
     // ensure the feed node feeds the existing mergemem;
     for (DUIterator_Fast imax, i = feed->fast_outs(imax); i < imax; i++) {
       x = feed->fast_out(i);
-      if (x == mm) {
+      if (x == mm || (UseShenandoahGC && x == mm2)) {
         break;
       }
     }
+
     if (x != mm) {
-      return NULL;
+      if (!UseShenandoahGC) {
+        // mm was our only chance
+        return NULL;
+      } else if (x != mm2) {
+        // mm2 was our only other chance
+        return NULL;
+      } else {
+        // mm2 is the merge fed by the store
+        // search from there for the membar
+        mm = mm2;
+      }
     }
 
     MemBarNode *mbar = NULL;
     // ensure the merge feeds to the expected type of membar
     for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {

@@ -4348,10 +4383,19 @@
                Assembler::byte, /*acquire*/ false, /*release*/ true,
                /*weak*/ false, noreg);
   %}
 
 
+  enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
+    MacroAssembler _masm(&cbuf);
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
+  %}
+
   // The only difference between aarch64_enc_cmpxchg and
   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
   // CompareAndSwap sequence to serve as a barrier on acquiring a
   // lock.
   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{

@@ -4369,10 +4413,20 @@
                Assembler::word, /*acquire*/ true, /*release*/ true,
                /*weak*/ false, noreg);
   %}
 
 
+  enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
+    MacroAssembler _masm(&cbuf);
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false,
+                                                   $res$$Register);
+  %}
+
   // auxiliary used for CompareAndSwapX to set result register
   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
     MacroAssembler _masm(&cbuf);
     Register res_reg = as_Register($res$$reg);
     __ cset(res_reg, Assembler::EQ);

@@ -9289,10 +9343,25 @@
   %}
 
   ins_pipe(ialu_reg);
 %}
 
+instruct shenandoahRB(iRegPNoSp dst, iRegP src, rFlagsReg cr) %{
+  match(Set dst (ShenandoahReadBarrier src));
+  format %{ "shenandoah_rb $dst,$src" %}
+  ins_encode %{
+#if INCLUDE_SHENANDOAHGC
+    Register s = $src$$Register;
+    Register d = $dst$$Register;
+    __ ldr(d, Address(s, ShenandoahBrooksPointer::byte_offset()));
+#else
+    ShouldNotReachHere();
+#endif
+  %}
+  ins_pipe(pipe_class_memory);
+%}
+
 
 // Convert oop pointer into compressed form
 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
   match(Set dst (EncodeP src));

@@ -9599,10 +9668,11 @@
   ins_pipe(pipe_slow);
 %}
 
 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 
+  predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   ins_cost(2 * VOLATILE_REF_COST);
 
   effect(KILL cr);
 

@@ -9615,12 +9685,30 @@
             aarch64_enc_cset_eq(res));
 
   ins_pipe(pipe_slow);
 %}
 
+instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+
+  predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchg_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+
+  ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp, res));
+
+  ins_pipe(pipe_slow);
+%}
+
 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 
+  predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR);
   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
   ins_cost(2 * VOLATILE_REF_COST);
 
   effect(KILL cr);
 

@@ -9633,10 +9721,31 @@
             aarch64_enc_cset_eq(res));
 
   ins_pipe(pipe_slow);
 %}
 
+instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+
+  predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
+  match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchgw_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
 // alternative CompareAndSwapX when we are eliding barriers
 
 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 
   predicate(needs_acquiring_load_exclusive(n));

@@ -9675,11 +9784,11 @@
   ins_pipe(pipe_slow);
 %}
 
 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 
-  predicate(needs_acquiring_load_exclusive(n));
+  predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR));
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   ins_cost(VOLATILE_REF_COST);
 
   effect(KILL cr);
 

@@ -9692,13 +9801,30 @@
             aarch64_enc_cset_eq(res));
 
   ins_pipe(pipe_slow);
 %}
 
+instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+
+  predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+
+  ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp, res));
+
+  ins_pipe(pipe_slow);
+%}
+
 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 
-  predicate(needs_acquiring_load_exclusive(n));
+  predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier|| n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR));
   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
   ins_cost(VOLATILE_REF_COST);
 
   effect(KILL cr);
 

@@ -9711,10 +9837,30 @@
             aarch64_enc_cset_eq(res));
 
   ins_pipe(pipe_slow);
 %}
 
+instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+
+  predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
+  match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+ format %{
+    "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+ %}
+
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
+  %}
+
+  ins_pipe(pipe_slow);
+%}
 
 // ---------------------------------------------------------------------
 
 
 // BEGIN This section of the file is automatically generated. Do not edit --------------

@@ -9791,10 +9937,11 @@
   %}
   ins_pipe(pipe_slow);
 %}
 
 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
+  predicate(!UseShenandoahGC || !ShenandoahCASBarrier);
   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
   ins_cost(2 * VOLATILE_REF_COST);
   effect(TEMP_DEF res, KILL cr);
   format %{
     "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"

@@ -9805,11 +9952,29 @@
                /*weak*/ false, $res$$Register);
   %}
   ins_pipe(pipe_slow);
 %}
 
+instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  predicate(UseShenandoahGC && ShenandoahCASBarrier);
+  match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
+  ins_cost(3 * VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
   ins_cost(2 * VOLATILE_REF_COST);
   effect(TEMP_DEF res, KILL cr);
   format %{
     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"

@@ -9820,10 +9985,27 @@
                /*weak*/ false, $res$$Register);
   %}
   ins_pipe(pipe_slow);
 %}
 
+instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  predicate(UseShenandoahGC && ShenandoahCASBarrier);
+  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+  ins_cost(3 * VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
   ins_cost(2 * VOLATILE_REF_COST);
   effect(KILL cr);
   format %{

@@ -9889,10 +10071,11 @@
   %}
   ins_pipe(pipe_slow);
 %}
 
 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
+  predicate(!UseShenandoahGC || !ShenandoahCASBarrier);
   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
   ins_cost(2 * VOLATILE_REF_COST);
   effect(KILL cr);
   format %{
     "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"

@@ -9905,11 +10088,29 @@
     __ csetw($res$$Register, Assembler::EQ);
   %}
   ins_pipe(pipe_slow);
 %}
 
+instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  predicate(UseShenandoahGC && ShenandoahCASBarrier);
+  match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(3 * VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                   /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
   ins_cost(2 * VOLATILE_REF_COST);
   effect(KILL cr);
   format %{
     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"

@@ -9922,10 +10123,26 @@
     __ csetw($res$$Register, Assembler::EQ);
   %}
   ins_pipe(pipe_slow);
 %}
 
+instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  predicate(UseShenandoahGC && ShenandoahCASBarrier);
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(3 * VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                   /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
 // END This section of the file is automatically generated. Do not edit --------------
 // ---------------------------------------------------------------------
 
 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
   match(Set prev (GetAndSetI mem newv));
< prev index next >