< prev index next >

src/hotspot/share/opto/arraycopynode.cpp

Print this page

        

@@ -25,13 +25,17 @@
 #include "precompiled.hpp"
 #include "gc/shared/barrierSet.hpp"
 #include "gc/shared/c2/barrierSetC2.hpp"
 #include "gc/shared/c2/cardTableBarrierSetC2.hpp"
 #include "opto/arraycopynode.hpp"
+#include "opto/castnode.hpp"
 #include "opto/graphKit.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "utilities/macros.hpp"
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
+#endif
 
 ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled, bool has_negative_length_guard)
   : CallNode(arraycopy_type(), NULL, TypeRawPtr::BOTTOM),
     _alloc_tightly_coupled(alloc_tightly_coupled),
     _has_negative_length_guard(has_negative_length_guard),

@@ -146,10 +150,56 @@
   }
 
   return get_length_if_constant(phase);
 }
 
+#if INCLUDE_SHENANDOAHGC
+Node* ArrayCopyNode::shenandoah_add_storeval_barrier(PhaseGVN *phase, bool can_reshape, Node* v, MergeMemNode* mem, Node*& ctl) {
+  if (ShenandoahStoreValReadBarrier) {
+    RegionNode* region = new RegionNode(3);
+    const Type* v_t = phase->type(v);
+    Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t);
+    Node* cmp = phase->transform(new CmpPNode(v, phase->zerocon(T_OBJECT)));
+    Node* bol = phase->transform(new BoolNode(cmp, BoolTest::ne));
+    IfNode* iff = new IfNode(ctl, bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN);
+
+    phase->transform(iff);
+    if (can_reshape) {
+      phase->is_IterGVN()->_worklist.push(iff);
+    } else {
+      phase->record_for_igvn(iff);
+    }
+
+    Node* null_true = phase->transform(new IfFalseNode(iff));
+    Node* null_false = phase->transform(new IfTrueNode(iff));
+    region->init_req(1, null_true);
+    region->init_req(2, null_false);
+    phi->init_req(1, phase->zerocon(T_OBJECT));
+    Node* cast = new CastPPNode(v, phase->type(v)->join_speculative(TypePtr::NOTNULL));
+    cast->set_req(0, null_false);
+    cast = phase->transform(cast);
+    Node* rb = phase->transform(new ShenandoahReadBarrierNode(null_false, phase->C->immutable_memory(), cast, false));
+    phi->init_req(2, rb);
+    ctl = phase->transform(region);
+    return phase->transform(phi);
+  }
+  if (ShenandoahStoreValEnqueueBarrier) {
+    const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(phase->type(v));
+    int alias = phase->C->get_alias_index(adr_type);
+    Node* wb = new ShenandoahWriteBarrierNode(phase->C, ctl, mem->memory_at(alias), v);
+    Node* wb_transformed = phase->transform(wb);
+    Node* enqueue = phase->transform(new ShenandoahEnqueueBarrierNode(wb_transformed));
+    if (wb_transformed == wb) {
+      Node* proj = phase->transform(new ShenandoahWBMemProjNode(wb));
+      mem->set_memory_at(alias, proj);
+    }
+    return enqueue;
+  }
+  return v;
+}
+#endif
+
 Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) {
   if (!is_clonebasic()) {
     return NULL;
   }
 

@@ -203,10 +253,15 @@
       type = Type::get_const_basic_type(bt);
     }
 
     Node* v = LoadNode::make(*phase, ctl, mem->memory_at(fieldidx), next_src, adr_type, type, bt, MemNode::unordered);
     v = phase->transform(v);
+#if INCLUDE_SHENANDOAHGC
+    if (UseShenandoahGC && bt == T_OBJECT) {
+      v = shenandoah_add_storeval_barrier(phase, can_reshape, v, mem, ctl);
+    }
+#endif
     Node* s = StoreNode::make(*phase, ctl, mem->memory_at(fieldidx), next_dest, adr_type, v, bt, MemNode::unordered);
     s = phase->transform(s);
     mem->set_memory_at(fieldidx, s);
   }
 

@@ -256,11 +311,11 @@
       return false;
     }
 
     BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
     if (dest_elem == T_OBJECT && (!is_alloc_tightly_coupled() ||
-                                  bs->array_copy_requires_gc_barriers(T_OBJECT))) {
+         (bs->array_copy_requires_gc_barriers(T_OBJECT) SHENANDOAHGC_ONLY(&& !ShenandoahStoreValEnqueueBarrier)))) {
       // It's an object array copy but we can't emit the card marking
       // that is needed
       return false;
     }
 

@@ -348,96 +403,124 @@
   }
 }
 
 Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase,
                                         bool can_reshape,
-                                        Node* forward_ctl,
-                                        Node* start_mem_src,
-                                        Node* start_mem_dest,
+                                        Node*& forward_ctl,
+                                        MergeMemNode* mm,
                                         const TypePtr* atp_src,
                                         const TypePtr* atp_dest,
                                         Node* adr_src,
                                         Node* base_src,
                                         Node* adr_dest,
                                         Node* base_dest,
                                         BasicType copy_type,
                                         const Type* value_type,
                                         int count) {
-  Node* mem = phase->C->top();
   if (!forward_ctl->is_top()) {
     // copy forward
-    mem = start_mem_dest;
+    mm = mm->clone()->as_MergeMem();
     uint alias_idx_src = phase->C->get_alias_index(atp_src);
     uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
+    Node *start_mem_src = mm->memory_at(alias_idx_src);
+    Node *start_mem_dest = mm->memory_at(alias_idx_dest);
+    Node* mem = start_mem_dest;
     bool same_alias = (alias_idx_src == alias_idx_dest);
 
     if (count > 0) {
       Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
       v = phase->transform(v);
+#if INCLUDE_SHENANDOAHGC
+      if (UseShenandoahGC && copy_type == T_OBJECT) {
+        v = shenandoah_add_storeval_barrier(phase, can_reshape, v, mm, forward_ctl);
+      }
+#endif
       mem = StoreNode::make(*phase, forward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
       mem = phase->transform(mem);
       for (int i = 1; i < count; i++) {
         Node* off  = phase->MakeConX(type2aelembytes(copy_type) * i);
         Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
         Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
         v = LoadNode::make(*phase, forward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
         v = phase->transform(v);
+#if INCLUDE_SHENANDOAHGC
+        if (UseShenandoahGC && copy_type == T_OBJECT) {
+          v = shenandoah_add_storeval_barrier(phase, can_reshape, v, mm, forward_ctl);
+        }
+#endif
         mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
         mem = phase->transform(mem);
       }
+      mm->set_memory_at(alias_idx_dest, mem);
     } else if(can_reshape) {
       PhaseIterGVN* igvn = phase->is_IterGVN();
       igvn->_worklist.push(adr_src);
       igvn->_worklist.push(adr_dest);
     }
+    return mm;
   }
-  return mem;
+  return phase->C->top();
 }
 
 Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase,
                                          bool can_reshape,
-                                         Node* backward_ctl,
-                                         Node* start_mem_src,
-                                         Node* start_mem_dest,
+                                         Node*& backward_ctl,
+                                         MergeMemNode* mm,
                                          const TypePtr* atp_src,
                                          const TypePtr* atp_dest,
                                          Node* adr_src,
                                          Node* base_src,
                                          Node* adr_dest,
                                          Node* base_dest,
                                          BasicType copy_type,
                                          const Type* value_type,
                                          int count) {
-  Node* mem = phase->C->top();
   if (!backward_ctl->is_top()) {
     // copy backward
-    mem = start_mem_dest;
+    mm = mm->clone()->as_MergeMem();
     uint alias_idx_src = phase->C->get_alias_index(atp_src);
     uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
+    Node *start_mem_src = mm->memory_at(alias_idx_src);
+    Node *start_mem_dest = mm->memory_at(alias_idx_dest);
+    Node* mem = start_mem_dest;
+
+    assert(copy_type != T_OBJECT SHENANDOAHGC_ONLY(|| ShenandoahStoreValEnqueueBarrier), "only tightly coupled allocations for object arrays");
     bool same_alias = (alias_idx_src == alias_idx_dest);
 
     if (count > 0) {
       for (int i = count-1; i >= 1; i--) {
         Node* off  = phase->MakeConX(type2aelembytes(copy_type) * i);
         Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
         Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
         Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
         v = phase->transform(v);
+#if INCLUDE_SHENANDOAHGC
+        if (UseShenandoahGC && copy_type == T_OBJECT) {
+          v = shenandoah_add_storeval_barrier(phase, can_reshape, v, mm, backward_ctl);
+        }
+#endif
         mem = StoreNode::make(*phase, backward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
         mem = phase->transform(mem);
       }
       Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
       v = phase->transform(v);
+#if INCLUDE_SHENANDOAHGC
+      if (UseShenandoahGC && copy_type == T_OBJECT) {
+        v = shenandoah_add_storeval_barrier(phase, can_reshape, v, mm, backward_ctl);
+      }
+#endif
       mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
       mem = phase->transform(mem);
+      mm->set_memory_at(alias_idx_dest, mem);
     } else if(can_reshape) {
       PhaseIterGVN* igvn = phase->is_IterGVN();
       igvn->_worklist.push(adr_src);
       igvn->_worklist.push(adr_dest);
     }
+    return phase->transform(mm);
   }
-  return mem;
+  return phase->C->top();
 }
 
 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,
                                      Node* ctl, Node *mem) {
   if (can_reshape) {

@@ -483,11 +566,12 @@
       remove_dead_region(phase, can_reshape);
     }
   } else {
     if (in(TypeFunc::Control) != ctl) {
       // we can't return new memory and control from Ideal at parse time
-      assert(!is_clonebasic(), "added control for clone?");
+      assert(!is_clonebasic() || UseShenandoahGC, "added control for clone?");
+      phase->record_for_igvn(this);
       return false;
     }
   }
   return true;
 }

@@ -557,19 +641,14 @@
 
   Node* src = in(ArrayCopyNode::Src);
   Node* dest = in(ArrayCopyNode::Dest);
   const TypePtr* atp_src = get_address_type(phase, src);
   const TypePtr* atp_dest = get_address_type(phase, dest);
-  uint alias_idx_src = phase->C->get_alias_index(atp_src);
-  uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
 
   Node *in_mem = in(TypeFunc::Memory);
-  Node *start_mem_src = in_mem;
-  Node *start_mem_dest = in_mem;
-  if (in_mem->is_MergeMem()) {
-    start_mem_src = in_mem->as_MergeMem()->memory_at(alias_idx_src);
-    start_mem_dest = in_mem->as_MergeMem()->memory_at(alias_idx_dest);
+  if (!in_mem->is_MergeMem()) {
+    in_mem = MergeMemNode::make(in_mem);
   }
 
 
   if (can_reshape) {
     assert(!phase->is_IterGVN()->delay_transform(), "cannot delay transforms");

@@ -579,31 +658,39 @@
   Node* backward_ctl = phase->C->top();
   Node* forward_ctl = phase->C->top();
   array_copy_test_overlap(phase, can_reshape, disjoint_bases, count, forward_ctl, backward_ctl);
 
   Node* forward_mem = array_copy_forward(phase, can_reshape, forward_ctl,
-                                         start_mem_src, start_mem_dest,
+                                         in_mem->as_MergeMem(),
                                          atp_src, atp_dest,
                                          adr_src, base_src, adr_dest, base_dest,
                                          copy_type, value_type, count);
 
   Node* backward_mem = array_copy_backward(phase, can_reshape, backward_ctl,
-                                           start_mem_src, start_mem_dest,
+                                           in_mem->as_MergeMem(),
                                            atp_src, atp_dest,
                                            adr_src, base_src, adr_dest, base_dest,
                                            copy_type, value_type, count);
 
   Node* ctl = NULL;
   if (!forward_ctl->is_top() && !backward_ctl->is_top()) {
     ctl = new RegionNode(3);
-    mem = new PhiNode(ctl, Type::MEMORY, atp_dest);
     ctl->init_req(1, forward_ctl);
-    mem->init_req(1, forward_mem);
     ctl->init_req(2, backward_ctl);
-    mem->init_req(2, backward_mem);
     ctl = phase->transform(ctl);
-    mem = phase->transform(mem);
+    MergeMemNode* forward_mm = forward_mem->as_MergeMem();
+    MergeMemNode* backward_mm = backward_mem->as_MergeMem();
+    for (MergeMemStream mms(forward_mm, backward_mm); mms.next_non_empty2(); ) {
+      if (mms.memory() != mms.memory2()) {
+        Node* phi = new PhiNode(ctl, Type::MEMORY, phase->C->get_adr_type(mms.alias_idx()));
+        phi->init_req(1, mms.memory());
+        phi->init_req(2, mms.memory2());
+        phi = phase->transform(phi);
+        mms.set_memory(phi);
+      }
+    }
+    mem = forward_mem;
   } else if (!forward_ctl->is_top()) {
     ctl = forward_ctl;
     mem = forward_mem;
   } else {
     assert(!backward_ctl->is_top(), "no copy?");

@@ -614,14 +701,10 @@
   if (can_reshape) {
     assert(phase->is_IterGVN()->delay_transform(), "should be delaying transforms");
     phase->is_IterGVN()->set_delay_transform(false);
   }
 
-  MergeMemNode* out_mem = MergeMemNode::make(in_mem);
-  out_mem->set_memory_at(alias_idx_dest, mem);
-  mem = out_mem;
-
   if (!finish_transform(phase, can_reshape, ctl, mem)) {
     return NULL;
   }
 
   return mem;
< prev index next >