< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp

Print this page

        

@@ -1,7 +1,7 @@
 /*
- * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
  *
  * This code is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 only, as
  * published by the Free Software Foundation.
  *

@@ -20,13 +20,15 @@
  * questions.
  *
  */
 
 #include "precompiled.hpp"
+#include "gc/shared/barrierSet.hpp"
 #include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shenandoah/shenandoahHeuristics.hpp"
 #include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
 #include "opto/arraycopynode.hpp"
 #include "opto/escape.hpp"
 #include "opto/graphKit.hpp"

@@ -39,125 +41,60 @@
 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
   return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
 }
 
 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
-  : _shenandoah_barriers(new (comp_arena) GrowableArray<ShenandoahWriteBarrierNode*>(comp_arena, 8,  0, NULL)) {
+  : _enqueue_barriers(new (comp_arena) GrowableArray<ShenandoahEnqueueBarrierNode*>(comp_arena, 8,  0, NULL)),
+    _load_reference_barriers(new (comp_arena) GrowableArray<ShenandoahLoadReferenceBarrierNode*>(comp_arena, 8,  0, NULL)) {
 }
 
-int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const {
-  return _shenandoah_barriers->length();
+int ShenandoahBarrierSetC2State::enqueue_barriers_count() const {
+  return _enqueue_barriers->length();
 }
 
-ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const {
-  return _shenandoah_barriers->at(idx);
+ShenandoahEnqueueBarrierNode* ShenandoahBarrierSetC2State::enqueue_barrier(int idx) const {
+  return _enqueue_barriers->at(idx);
 }
 
-void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {
-  assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list");
-  _shenandoah_barriers->append(n);
+void ShenandoahBarrierSetC2State::add_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) {
+  assert(!_enqueue_barriers->contains(n), "duplicate entry in barrier list");
+  _enqueue_barriers->append(n);
 }
 
-void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {
-  if (_shenandoah_barriers->contains(n)) {
-    _shenandoah_barriers->remove(n);
+void ShenandoahBarrierSetC2State::remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) {
+  if (_enqueue_barriers->contains(n)) {
+    _enqueue_barriers->remove(n);
   }
 }
 
-#define __ kit->
-
-Node* ShenandoahBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const {
-  if (ShenandoahReadBarrier) {
-    obj = shenandoah_read_barrier_impl(kit, obj, false, true, true);
-  }
-  return obj;
+int ShenandoahBarrierSetC2State::load_reference_barriers_count() const {
+  return _load_reference_barriers->length();
 }
 
-Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const {
-  if (ShenandoahStoreValEnqueueBarrier) {
-    obj = shenandoah_write_barrier(kit, obj);
-    obj = shenandoah_enqueue_barrier(kit, obj);
-  }
-  if (ShenandoahStoreValReadBarrier) {
-    obj = shenandoah_read_barrier_impl(kit, obj, true, false, false);
-  }
-  return obj;
+ShenandoahLoadReferenceBarrierNode* ShenandoahBarrierSetC2State::load_reference_barrier(int idx) const {
+  return _load_reference_barriers->at(idx);
 }
 
-Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const {
-  const Type* obj_type = obj->bottom_type();
-  if (obj_type->higher_equal(TypePtr::NULL_PTR)) {
-    return obj;
-  }
-  const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
-  Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory();
-
-  if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) {
-    // We know it is null, no barrier needed.
-    return obj;
-  }
-
-  if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) {
-
-    // We don't know if it's null or not. Need null-check.
-    enum { _not_null_path = 1, _null_path, PATH_LIMIT };
-    RegionNode* region = new RegionNode(PATH_LIMIT);
-    Node*       phi    = new PhiNode(region, obj_type);
-    Node* null_ctrl = __ top();
-    Node* not_null_obj = __ null_check_oop(obj, &null_ctrl);
-
-    region->init_req(_null_path, null_ctrl);
-    phi   ->init_req(_null_path, __ zerocon(T_OBJECT));
-
-    Node* ctrl = use_ctrl ? __ control() : NULL;
-    ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace);
-    Node* n = __ gvn().transform(rb);
-
-    region->init_req(_not_null_path, __ control());
-    phi   ->init_req(_not_null_path, n);
-
-    __ set_control(__ gvn().transform(region));
-    __ record_for_igvn(region);
-    return __ gvn().transform(phi);
-
-  } else {
-    // We know it is not null. Simple barrier is sufficient.
-    Node* ctrl = use_ctrl ? __ control() : NULL;
-    ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace);
-    Node* n = __ gvn().transform(rb);
-    __ record_for_igvn(n);
-    return n;
-  }
+void ShenandoahBarrierSetC2State::add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) {
+  assert(!_load_reference_barriers->contains(n), "duplicate entry in barrier list");
+  _load_reference_barriers->append(n);
 }
 
-Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const {
-  ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj);
-  Node* n = __ gvn().transform(wb);
-  if (n == wb) { // New barrier needs memory projection.
-    Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n));
-    __ set_memory(proj, adr_type);
+void ShenandoahBarrierSetC2State::remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) {
+  if (_load_reference_barriers->contains(n)) {
+    _load_reference_barriers->remove(n);
   }
-  return n;
 }
 
-Node* ShenandoahBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const {
-  if (ShenandoahWriteBarrier) {
-    obj = shenandoah_write_barrier_impl(kit, obj);
+Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const {
+  if (ShenandoahStoreValEnqueueBarrier) {
+    obj = shenandoah_enqueue_barrier(kit, obj);
   }
   return obj;
 }
 
-Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const {
-  if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) {
-    return obj;
-  }
-  const Type* obj_type = obj->bottom_type();
-  const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
-  Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type);
-  __ record_for_igvn(n);
-  return n;
-}
+#define __ kit->
 
 bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr,
                                                          BasicType bt, uint adr_idx) const {
   intptr_t offset = 0;
   Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);

@@ -300,11 +237,11 @@
   // Now some of the values
   Node* marking;
   Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
   Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
   marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING));
-  assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape");
+  assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape");
 
   // if (!marking)
   __ if_then(marking, BoolTest::ne, zero, unlikely); {
     BasicType index_bt = TypeX_X->basic_type();
     assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");

@@ -357,11 +294,11 @@
          call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);
 }
 
 bool ShenandoahBarrierSetC2::is_shenandoah_wb_call(Node* call) {
   return call->is_CallLeaf() &&
-         call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT);
+         call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT);
 }
 
 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
   if (n->Opcode() != Op_If) {
     return false;

@@ -545,92 +482,10 @@
   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
 
   return TypeFunc::make(domain, range);
 }
 
-void ShenandoahBarrierSetC2::resolve_address(C2Access& access) const {
-  const TypePtr* adr_type = access.addr().type();
-
-  if ((access.decorators() & IN_NATIVE) == 0 && (adr_type->isa_instptr() || adr_type->isa_aryptr())) {
-    int off = adr_type->is_ptr()->offset();
-    int base_off = adr_type->isa_instptr() ? instanceOopDesc::base_offset_in_bytes() :
-      arrayOopDesc::base_offset_in_bytes(adr_type->is_aryptr()->elem()->array_element_basic_type());
-    assert(off != Type::OffsetTop, "unexpected offset");
-    if (off == Type::OffsetBot || off >= base_off) {
-      DecoratorSet decorators = access.decorators();
-      bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
-      GraphKit* kit = NULL;
-      if (access.is_parse_access()) {
-        C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
-        kit = parse_access.kit();
-      }
-      Node* adr = access.addr().node();
-      assert(adr->is_AddP(), "unexpected address shape");
-      Node* base = adr->in(AddPNode::Base);
-
-      if (is_write) {
-        if (kit != NULL) {
-          base = shenandoah_write_barrier(kit, base);
-        } else {
-          assert(access.is_opt_access(), "either parse or opt access");
-          assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for clone");
-        }
-      } else {
-        if (adr_type->isa_instptr()) {
-          Compile* C = access.gvn().C;
-          ciField* field = C->alias_type(adr_type)->field();
-
-          // Insert read barrier for Shenandoah.
-          if (field != NULL &&
-              ((ShenandoahOptimizeStaticFinals   && field->is_static()  && field->is_final()) ||
-               (ShenandoahOptimizeInstanceFinals && !field->is_static() && field->is_final()) ||
-               (ShenandoahOptimizeStableFinals   && field->is_stable()))) {
-            // Skip the barrier for special fields
-          } else {
-            if (kit != NULL) {
-              base = shenandoah_read_barrier(kit, base);
-            } else {
-              assert(access.is_opt_access(), "either parse or opt access");
-              assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
-            }
-          }
-        } else {
-          if (kit != NULL) {
-            base = shenandoah_read_barrier(kit, base);
-          } else {
-            assert(access.is_opt_access(), "either parse or opt access");
-            assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
-          }
-        }
-      }
-      if (base != adr->in(AddPNode::Base)) {
-        assert(kit != NULL, "no barrier should have been added");
-
-        Node* address = adr->in(AddPNode::Address);
-
-        if (address->is_AddP()) {
-          assert(address->in(AddPNode::Base) == adr->in(AddPNode::Base), "unexpected address shape");
-          assert(!address->in(AddPNode::Address)->is_AddP(), "unexpected address shape");
-          assert(address->in(AddPNode::Address) == adr->in(AddPNode::Base), "unexpected address shape");
-          address = address->clone();
-          address->set_req(AddPNode::Base, base);
-          address->set_req(AddPNode::Address, base);
-          address = kit->gvn().transform(address);
-        } else {
-          assert(address == adr->in(AddPNode::Base), "unexpected address shape");
-          address = base;
-        }
-        adr = adr->clone();
-        adr->set_req(AddPNode::Base, base);
-        adr->set_req(AddPNode::Address, address);
-        adr = kit->gvn().transform(adr);
-        access.addr().set_node(adr);
-      }
-    }
-  }
-}
-
 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
   DecoratorSet decorators = access.decorators();
 
   const TypePtr* adr_type = access.addr().type();
   Node* adr = access.addr().node();

@@ -653,53 +508,17 @@
     val.set_node(value);
     shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
                                  static_cast<const TypeOopPtr*>(val.type()), NULL /* pre_val */, access.type());
   } else {
     assert(access.is_opt_access(), "only for optimization passes");
-    assert(((decorators & C2_TIGHLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
+    assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
     C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
     PhaseGVN& gvn =  opt_access.gvn();
     MergeMemNode* mm = opt_access.mem();
 
-    if (ShenandoahStoreValReadBarrier) {
-      RegionNode* region = new RegionNode(3);
-      const Type* v_t = gvn.type(val.node());
-      Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t);
-      Node* cmp = gvn.transform(new CmpPNode(val.node(), gvn.zerocon(T_OBJECT)));
-      Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne));
-      IfNode* iff = new IfNode(opt_access.ctl(), bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN);
-
-      gvn.transform(iff);
-      if (gvn.is_IterGVN()) {
-        gvn.is_IterGVN()->_worklist.push(iff);
-      } else {
-        gvn.record_for_igvn(iff);
-      }
-
-      Node* null_true = gvn.transform(new IfFalseNode(iff));
-      Node* null_false = gvn.transform(new IfTrueNode(iff));
-      region->init_req(1, null_true);
-      region->init_req(2, null_false);
-      phi->init_req(1, gvn.zerocon(T_OBJECT));
-      Node* cast = new CastPPNode(val.node(), gvn.type(val.node())->join_speculative(TypePtr::NOTNULL));
-      cast->set_req(0, null_false);
-      cast = gvn.transform(cast);
-      Node* rb = gvn.transform(new ShenandoahReadBarrierNode(null_false, gvn.C->immutable_memory(), cast, false));
-      phi->init_req(2, rb);
-      opt_access.set_ctl(gvn.transform(region));
-      val.set_node(gvn.transform(phi));
-    }
     if (ShenandoahStoreValEnqueueBarrier) {
-      const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(gvn.type(val.node()));
-      int alias = gvn.C->get_alias_index(adr_type);
-      Node* wb = new ShenandoahWriteBarrierNode(gvn.C, opt_access.ctl(), mm->memory_at(alias), val.node());
-      Node* wb_transformed = gvn.transform(wb);
-      Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(wb_transformed));
-      if (wb_transformed == wb) {
-        Node* proj = gvn.transform(new ShenandoahWBMemProjNode(wb));
-        mm->set_memory_at(alias, proj);
-      }
+      Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(val.node()));
       val.set_node(enqueue);
     }
   }
   return BarrierSetC2::store_at_resolved(access, val);
 }

@@ -720,10 +539,21 @@
   Node* top = Compile::current()->top();
 
   Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
   Node* load = BarrierSetC2::load_at_resolved(access, val_type);
 
+  if (access.is_oop()) {
+    if (ShenandoahLoadRefBarrier) {
+      load = new ShenandoahLoadReferenceBarrierNode(NULL, load);
+      if (access.is_parse_access()) {
+        load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
+      } else {
+        load = static_cast<C2OptAccess &>(access).gvn().transform(load);
+      }
+    }
+  }
+
   // If we are reading the value of the referent field of a Reference
   // object (either by using Unsafe directly or through reflection)
   // then, if SATB is enabled, we need to record the referent in an
   // SATB log buffer using the pre-barrier mechanism.
   // Also we need to add memory barrier to prevent commoning reads

@@ -773,25 +603,34 @@
 
 #ifdef _LP64
     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
       Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
       Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
-      load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
+      if (ShenandoahCASBarrier) {
+        load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
+      } else {
+        load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
+      }
     } else
 #endif
     {
-      load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
+      if (ShenandoahCASBarrier) {
+        load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
+      } else {
+        load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
+      }
     }
 
     access.set_raw_access(load_store);
     pin_atomic_op(access);
 
 #ifdef _LP64
     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
-      return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
+      load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
     }
 #endif
+    load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store));
     return load_store;
   }
   return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
 }
 

@@ -811,22 +650,38 @@
     Node* adr = access.addr().node();
 #ifdef _LP64
     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
       Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
       Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
-      if (is_weak_cas) {
-        load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+      if (ShenandoahCASBarrier) {
+        if (is_weak_cas) {
+          load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+        } else {
+          load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+        }
       } else {
-        load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+        if (is_weak_cas) {
+          load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+        } else {
+          load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
+        }
       }
     } else
 #endif
     {
-      if (is_weak_cas) {
-        load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+      if (ShenandoahCASBarrier) {
+        if (is_weak_cas) {
+          load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+        } else {
+          load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+        }
       } else {
-        load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+        if (is_weak_cas) {
+          load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+        } else {
+          load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
+        }
       }
     }
     access.set_raw_access(load_store);
     pin_atomic_op(access);
     return load_store;

@@ -839,32 +694,23 @@
   if (access.is_oop()) {
     val = shenandoah_storeval_barrier(kit, val);
   }
   Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
   if (access.is_oop()) {
+    result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result));
     shenandoah_write_barrier_pre(kit, false /* do_load */,
                                  NULL, NULL, max_juint, NULL, NULL,
                                  result /* pre_val */, T_OBJECT);
   }
   return result;
 }
 
 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
   assert(!src->is_AddP(), "unexpected input");
-  src = shenandoah_read_barrier(kit, src);
   BarrierSetC2::clone(kit, src, dst, size, is_array);
 }
 
-Node* ShenandoahBarrierSetC2::resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const {
-  bool is_write = decorators & ACCESS_WRITE;
-  if (is_write) {
-    return shenandoah_write_barrier(kit, n);
-  } else {
-  return shenandoah_read_barrier(kit, n);
-  }
-}
-
 Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
                                            Node*& i_o, Node*& needgc_ctrl,
                                            Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
                                            intx prefetch_lines) const {
   PhaseIterGVN& igvn = macro->igvn();

@@ -887,10 +733,11 @@
   return fast_oop;
 }
 
 // Support for GC barriers emitted during parsing
 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
+  if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true;
   if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
     return false;
   }
   CallLeafNode *call = node->as_CallLeaf();
   if (call->_name == NULL) {

@@ -901,39 +748,42 @@
          strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
          strcmp(call->_name, "shenandoah_wb_pre") == 0;
 }
 
 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
-  return ShenandoahBarrierNode::skip_through_barrier(c);
+  if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+    return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
+  }
+  if (c->Opcode() == Op_ShenandoahEnqueueBarrier) {
+    c = c->in(1);
+  }
+  return c;
 }
 
 bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
-  return !ShenandoahWriteBarrierNode::expand(C, igvn);
+  return !ShenandoahBarrierC2Support::expand(C, igvn);
 }
 
 bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
   if (mode == LoopOptsShenandoahExpand) {
     assert(UseShenandoahGC, "only for shenandoah");
-    ShenandoahWriteBarrierNode::pin_and_expand(phase);
+    ShenandoahBarrierC2Support::pin_and_expand(phase);
     return true;
   } else if (mode == LoopOptsShenandoahPostExpand) {
     assert(UseShenandoahGC, "only for shenandoah");
     visited.Clear();
-    ShenandoahWriteBarrierNode::optimize_after_expansion(visited, nstack, worklist, phase);
+    ShenandoahBarrierC2Support::optimize_after_expansion(visited, nstack, worklist, phase);
     return true;
   }
-  GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
-  ShenandoahWriteBarrierNode::optimize_before_expansion(phase, memory_graph_fixers, false);
   return false;
 }
 
 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
   bool is_oop = type == T_OBJECT || type == T_ARRAY;
   if (!is_oop) {
     return false;
   }
-
   if (tightly_coupled_alloc) {
     if (phase == Optimization) {
       return false;
     }
     return !is_clone;

@@ -957,11 +807,11 @@
           igvn.C->dependencies()->assert_leaf_type(ik);
         }
       }
     } else {
       return true;
-    }
+        }
   } else if (src_type->isa_aryptr()) {
     BasicType src_elem  = src_type->klass()->as_array_klass()->element_type()->basic_type();
     if (src_elem == T_OBJECT || src_elem == T_ARRAY) {
       return true;
     }

@@ -1010,18 +860,24 @@
 }
 
 
 // Support for macro expanded GC barriers
 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
-  if (node->Opcode() == Op_ShenandoahWriteBarrier) {
-    state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);
+  if (node->Opcode() == Op_ShenandoahEnqueueBarrier) {
+    state()->add_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node);
+  }
+  if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+    state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
   }
 }
 
 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
-  if (node->Opcode() == Op_ShenandoahWriteBarrier) {
-    state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);
+  if (node->Opcode() == Op_ShenandoahEnqueueBarrier) {
+    state()->remove_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node);
+  }
+  if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+    state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
   }
 }
 
 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
   if (is_shenandoah_wb_pre_call(n)) {

@@ -1063,23 +919,22 @@
       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
         C->record_for_igvn(n->fast_out(i));
       }
     }
   }
-  for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) {
-    ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i);
+  for (int i = state()->enqueue_barriers_count() - 1; i >= 0; i--) {
+    ShenandoahEnqueueBarrierNode* n = state()->enqueue_barrier(i);
     if (!useful.member(n)) {
-      state()->remove_shenandoah_barrier(n);
+      state()->remove_enqueue_barrier(n);
+    }
+  }
+  for (int i = state()->load_reference_barriers_count() - 1; i >= 0; i--) {
+    ShenandoahLoadReferenceBarrierNode* n = state()->load_reference_barrier(i);
+    if (!useful.member(n)) {
+      state()->remove_load_reference_barrier(n);
     }
   }
-
-}
-
-bool ShenandoahBarrierSetC2::has_special_unique_user(const Node* node) const {
-  assert(node->outcnt() == 1, "match only for unique out");
-  Node* n = node->unique_out();
-  return node->Opcode() == Op_ShenandoahWriteBarrier && n->Opcode() == Op_ShenandoahWBMemProj;
 }
 
 void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {}
 
 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {

@@ -1095,11 +950,11 @@
 bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; }
 
 #ifdef ASSERT
 void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
   if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) {
-    ShenandoahBarrierNode::verify(Compile::current()->root());
+    ShenandoahBarrierC2Support::verify(Compile::current()->root());
   } else if (phase == BarrierSetC2::BeforeCodeGen) {
     // Verify G1 pre-barriers
     const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset());
 
     ResourceArea *area = Thread::current()->resource_area();

@@ -1201,19 +1056,19 @@
       }
       return n;
     }
   } else if (can_reshape &&
              n->Opcode() == Op_If &&
-             ShenandoahWriteBarrierNode::is_heap_stable_test(n) &&
+             ShenandoahBarrierC2Support::is_heap_stable_test(n) &&
              n->in(0) != NULL) {
     Node* dom = n->in(0);
     Node* prev_dom = n;
     int op = n->Opcode();
     int dist = 16;
     // Search up the dominator tree for another heap stable test
     while (dom->Opcode() != op    ||  // Not same opcode?
-           !ShenandoahWriteBarrierNode::is_heap_stable_test(dom) ||  // Not same input 1?
+           !ShenandoahBarrierC2Support::is_heap_stable_test(dom) ||  // Not same input 1?
            prev_dom->in(0) != dom) {  // One path of test does not dominate?
       if (dist < 0) return NULL;
 
       dist--;
       prev_dom = dom;

@@ -1230,74 +1085,20 @@
   }
 
   return NULL;
 }
 
-Node* ShenandoahBarrierSetC2::identity_node(PhaseGVN* phase, Node* n) const {
-  if (n->is_Load()) {
-    Node *mem = n->in(MemNode::Memory);
-    Node *value = n->as_Load()->can_see_stored_value(mem, phase);
-    if (value) {
-      PhaseIterGVN *igvn = phase->is_IterGVN();
-      if (igvn != NULL &&
-          value->is_Phi() &&
-          value->req() > 2 &&
-          value->in(1) != NULL &&
-          value->in(1)->is_ShenandoahBarrier()) {
-        if (igvn->_worklist.member(value) ||
-            igvn->_worklist.member(value->in(0)) ||
-            (value->in(0)->in(1) != NULL &&
-             value->in(0)->in(1)->is_IfProj() &&
-             (igvn->_worklist.member(value->in(0)->in(1)) ||
-              (value->in(0)->in(1)->in(0) != NULL &&
-               igvn->_worklist.member(value->in(0)->in(1)->in(0)))))) {
-          igvn->_worklist.push(n);
-          return n;
-        }
-      }
-      // (This works even when value is a Con, but LoadNode::Value
-      // usually runs first, producing the singleton type of the Con.)
-      Node *value_no_barrier = step_over_gc_barrier(value->Opcode() == Op_EncodeP ? value->in(1) : value);
-      if (value->Opcode() == Op_EncodeP) {
-        if (value_no_barrier != value->in(1)) {
-          Node *encode = value->clone();
-          encode->set_req(1, value_no_barrier);
-          encode = phase->transform(encode);
-          return encode;
-        }
-      } else {
-        return value_no_barrier;
-      }
-    }
-  }
-  return n;
-}
-
 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
     Node* u = n->fast_out(i);
     if (!is_shenandoah_wb_pre_call(u)) {
       return false;
     }
   }
   return n->outcnt() > 0;
 }
 
-bool ShenandoahBarrierSetC2::flatten_gc_alias_type(const TypePtr*& adr_type) const {
-  int offset = adr_type->offset();
-  if (offset == ShenandoahBrooksPointer::byte_offset()) {
-    if (adr_type->isa_aryptr()) {
-      adr_type = TypeAryPtr::make(adr_type->ptr(), adr_type->isa_aryptr()->ary(), adr_type->isa_aryptr()->klass(), false, offset);
-    } else if (adr_type->isa_instptr()) {
-      adr_type = TypeInstPtr::make(adr_type->ptr(), ciEnv::current()->Object_klass(), false, NULL, offset);
-    }
-    return true;
-  } else {
-    return false;
-  }
-}
-
 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
   switch (opcode) {
     case Op_CallLeaf:
     case Op_CallLeafNoFP: {
       assert (n->is_Call(), "");

@@ -1328,31 +1129,18 @@
         bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
         assert( !tp || oop_offset_is_sane, "" );
       }
 #endif
       return true;
-    case Op_ShenandoahReadBarrier:
-      return true;
-    case Op_ShenandoahWriteBarrier:
+    case Op_ShenandoahLoadReferenceBarrier:
       assert(false, "should have been expanded already");
       return true;
     default:
       return false;
   }
 }
 
-#ifdef ASSERT
-bool ShenandoahBarrierSetC2::verify_gc_alias_type(const TypePtr* adr_type, int offset) const {
-  if (offset == ShenandoahBrooksPointer::byte_offset() &&
-      (adr_type->base() == Type::AryPtr || adr_type->base() == Type::OopPtr)) {
-    return true;
-  } else {
-    return false;
-  }
-}
-#endif
-
 bool ShenandoahBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
   switch (opcode) {
     case Op_ShenandoahCompareAndExchangeP:
     case Op_ShenandoahCompareAndExchangeN:
       conn_graph->add_objload_to_connection_graph(n, delayed_worklist);

@@ -1384,19 +1172,16 @@
           }
         }
       }
       return false;
     }
-    case Op_ShenandoahReadBarrier:
-    case Op_ShenandoahWriteBarrier:
-      // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
-      // It doesn't escape.
-      conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), delayed_worklist);
-      break;
     case Op_ShenandoahEnqueueBarrier:
       conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist);
       break;
+    case Op_ShenandoahLoadReferenceBarrier:
+      conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), delayed_worklist);
+      return true;
     default:
       // Nothing
       break;
   }
   return false;

@@ -1413,19 +1198,16 @@
     case Op_ShenandoahCompareAndSwapP:
     case Op_ShenandoahCompareAndSwapN:
     case Op_ShenandoahWeakCompareAndSwapP:
     case Op_ShenandoahWeakCompareAndSwapN:
       return conn_graph->add_final_edges_unsafe_access(n, opcode);
-    case Op_ShenandoahReadBarrier:
-    case Op_ShenandoahWriteBarrier:
-      // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
-      // It doesn't escape.
-      conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), NULL);
-      return true;
     case Op_ShenandoahEnqueueBarrier:
       conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), NULL);
       return true;
+    case Op_ShenandoahLoadReferenceBarrier:
+      conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), NULL);
+      return true;
     default:
       // Nothing
       break;
   }
   return false;

@@ -1436,25 +1218,11 @@
          n->has_out_with(Op_ShenandoahCompareAndSwapP, Op_ShenandoahCompareAndSwapN, Op_ShenandoahWeakCompareAndSwapP, Op_ShenandoahWeakCompareAndSwapN);
 
 }
 
 bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const {
-  return n->is_ShenandoahBarrier();
-}
-
-bool ShenandoahBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
-  switch (opcode) {
-    case Op_ShenandoahReadBarrier:
-      if (n->in(ShenandoahBarrierNode::ValueIn)->is_DecodeNarrowPtr()) {
-        matcher->set_shared(n->in(ShenandoahBarrierNode::ValueIn)->in(1));
-      }
-      matcher->set_shared(n);
-      return true;
-    default:
-      break;
-  }
-  return false;
+  return n->Opcode() == Op_ShenandoahLoadReferenceBarrier;
 }
 
 bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const {
   switch (opcode) {
     case Op_ShenandoahCompareAndExchangeP:

@@ -1482,64 +1250,5 @@
          xop == Op_ShenandoahWeakCompareAndSwapP ||
          xop == Op_ShenandoahWeakCompareAndSwapN ||
          xop == Op_ShenandoahCompareAndSwapN ||
          xop == Op_ShenandoahCompareAndSwapP;
 }
-
-void ShenandoahBarrierSetC2::igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const {
-  if (use->is_ShenandoahBarrier()) {
-    for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
-      Node* u = use->fast_out(i2);
-      Node* cmp = use->find_out_with(Op_CmpP);
-      if (u->Opcode() == Op_CmpP) {
-        igvn->_worklist.push(cmp);
-      }
-    }
-  }
-}
-
-void ShenandoahBarrierSetC2::ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const {
-  if (use->is_ShenandoahBarrier()) {
-    for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
-      Node* p = use->fast_out(i2);
-      if (p->Opcode() == Op_AddP) {
-        for (DUIterator_Fast i3max, i3 = p->fast_outs(i3max); i3 < i3max; i3++) {
-          Node* q = p->fast_out(i3);
-          if (q->is_Load()) {
-            if(q->bottom_type() != ccp->type(q)) {
-              worklist.push(q);
-            }
-          }
-        }
-      }
-    }
-  }
-}
-
-Node* ShenandoahBarrierSetC2::split_if_pre(PhaseIdealLoop* phase, Node* n) const {
-  if (n->Opcode() == Op_ShenandoahReadBarrier) {
-    ((ShenandoahReadBarrierNode*)n)->try_move(phase);
-  } else if (n->Opcode() == Op_ShenandoahWriteBarrier) {
-    return ((ShenandoahWriteBarrierNode*)n)->try_split_thru_phi(phase);
-  }
-
-  return NULL;
-}
-
-bool ShenandoahBarrierSetC2::build_loop_late_post(PhaseIdealLoop* phase, Node* n) const {
-  return ShenandoahBarrierNode::build_loop_late_post(phase, n);
-}
-
-bool ShenandoahBarrierSetC2::sink_node(PhaseIdealLoop* phase, Node* n, Node* x, Node* x_ctrl, Node* n_ctrl) const {
-  if (n->is_ShenandoahBarrier()) {
-    return x->as_ShenandoahBarrier()->sink_node(phase, x_ctrl, n_ctrl);
-  }
-  if (n->is_MergeMem()) {
-    // PhaseIdealLoop::split_if_with_blocks_post() would:
-    // _igvn._worklist.yank(x);
-    // which sometimes causes chains of MergeMem which some of
-    // shenandoah specific code doesn't support
-    phase->register_new_node(x, x_ctrl);
-    return true;
-  }
-  return false;
-}
< prev index next >