< prev index next >

src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp

Print this page




   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "opto/castnode.hpp"
  26 #include "opto/compile.hpp"

  27 #include "opto/escape.hpp"
  28 #include "opto/graphKit.hpp"

  29 #include "opto/loopnode.hpp"
  30 #include "opto/machnode.hpp"
  31 #include "opto/macro.hpp"
  32 #include "opto/memnode.hpp"
  33 #include "opto/movenode.hpp"
  34 #include "opto/node.hpp"
  35 #include "opto/phase.hpp"
  36 #include "opto/phaseX.hpp"
  37 #include "opto/rootnode.hpp"
  38 #include "opto/type.hpp"
  39 #include "utilities/copy.hpp"
  40 #include "utilities/growableArray.hpp"
  41 #include "utilities/macros.hpp"
  42 #include "gc/z/zBarrierSet.hpp"
  43 #include "gc/z/c2/zBarrierSetC2.hpp"
  44 #include "gc/z/zThreadLocalData.hpp"
  45 #include "gc/z/zBarrierSetRuntime.hpp"
  46 
  47 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena) :
  48     _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8,  0, NULL)) {}
  49 
  50 int ZBarrierSetC2State::load_barrier_count() const {
  51   return _load_barrier_nodes->length();
  52 }
  53 
  54 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {
  55   assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");
  56   _load_barrier_nodes->append(n);
  57 }
  58 
  59 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {
  60   // this function may be called twice for a node so check


  74 
  75 ZBarrierSetC2State* ZBarrierSetC2::state() const {
  76   return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
  77 }
  78 
  79 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {
  80   // 1. This step follows potential oop projections of a load barrier before expansion
  81   if (node->is_Proj()) {
  82     node = node->in(0);
  83   }
  84 
  85   // 2. This step checks for unexpanded load barriers
  86   if (node->is_LoadBarrier()) {
  87     return true;
  88   }
  89 
  90   // 3. This step checks for the phi corresponding to an optimized load barrier expansion
  91   if (node->is_Phi()) {
  92     PhiNode* phi = node->as_Phi();
  93     Node* n = phi->in(1);
  94     if (n != NULL && n->is_LoadBarrierSlowReg()) {
  95       return true;
  96     }
  97   }
  98 
  99   return false;
 100 }
 101 
 102 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {
 103   if (node->is_LoadBarrier()) {
 104     state()->add_load_barrier_node(node->as_LoadBarrier());
 105   }
 106 }
 107 
 108 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
 109   if (node->is_LoadBarrier()) {
 110     state()->remove_load_barrier_node(node->as_LoadBarrier());
 111   }
 112 }
 113 
 114 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {
 115   // Remove useless LoadBarrier nodes
 116   ZBarrierSetC2State* s = state();
 117   for (int i = s->load_barrier_count()-1; i >= 0; i--) {
 118     LoadBarrierNode* n = s->load_barrier_node(i);
 119     if (!useful.member(n)) {
 120       unregister_potential_barrier_node(n);
 121     }
 122   }
 123 }
 124 
 125 void ZBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
 126   if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
 127     igvn->_worklist.push(node);
 128   }
 129 }
 130 
 131 static bool load_require_barrier(LoadNode* load)      { return ((load->barrier_data() & RequireBarrier) != 0); }
 132 static bool load_has_weak_barrier(LoadNode* load)     { return ((load->barrier_data() & WeakBarrier) != 0); }
 133 static bool load_has_expanded_barrier(LoadNode* load) { return ((load->barrier_data() & ExpandedBarrier) != 0); }
 134 static void load_set_expanded_barrier(LoadNode* load) { return load->set_barrier_data(ExpandedBarrier); }
 135 
 136 static void load_set_barrier(LoadNode* load, bool weak)    {
 137   if (weak) {
 138     load->set_barrier_data(WeakBarrier);
 139   } else {
 140     load->set_barrier_data(RequireBarrier);

 141   }
 142 }
 143 






























 144 // == LoadBarrierNode ==
 145 
 146 LoadBarrierNode::LoadBarrierNode(Compile* C,
 147                                  Node* c,
 148                                  Node* mem,
 149                                  Node* val,
 150                                  Node* adr,
 151                                  bool weak) :


 152     MultiNode(Number_of_Inputs),
 153     _weak(weak) {


 154   init_req(Control, c);
 155   init_req(Memory, mem);
 156   init_req(Oop, val);
 157   init_req(Address, adr);
 158   init_req(Similar, C->top());
 159 
 160   init_class_id(Class_LoadBarrier);
 161   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 162   bs->register_potential_barrier_node(this);
 163 }
 164 
 165 uint LoadBarrierNode::size_of() const {
 166   return sizeof(*this);
 167 }
 168 
 169 bool LoadBarrierNode::cmp(const Node& n) const {
 170   ShouldNotReachHere();
 171   return false;
 172 }
 173 
 174 const Type *LoadBarrierNode::bottom_type() const {
 175   const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
 176   Node* in_oop = in(Oop);
 177   floadbarrier[Control] = Type::CONTROL;
 178   floadbarrier[Memory] = Type::MEMORY;
 179   floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();
 180   return TypeTuple::make(Number_of_Outputs, floadbarrier);
 181 }
 182 
 183 const TypePtr* LoadBarrierNode::adr_type() const {
 184   ShouldNotReachHere();
 185   return NULL;
 186 }
 187 
 188 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
 189   const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
 190   const Type* val_t = phase->type(in(Oop));
 191   floadbarrier[Control] = Type::CONTROL;
 192   floadbarrier[Memory]  = Type::MEMORY;
 193   floadbarrier[Oop]     = val_t;
 194   return TypeTuple::make(Number_of_Outputs, floadbarrier);
 195 }
 196 
 197 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
 198   if (phase != NULL) {
 199     return phase->is_dominator(d, n);
 200   }
 201 
 202   for (int i = 0; i < 10 && n != NULL; i++) {
 203     n = IfNode::up_one_dom(n, linear_only);
 204     if (n == d) {
 205       return true;
 206     }
 207   }
 208 
 209   return false;
 210 }
 211 
 212 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {
 213   if (is_weak()) {
 214     // Weak barriers can't be eliminated
 215     return NULL;
 216   }
 217 
 218   Node* val = in(LoadBarrierNode::Oop);
 219   if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
 220     LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
 221     assert(lb->in(Address) == in(Address), "");
 222     // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
 223     if (lb->in(Oop) == in(Oop)) {
 224       return lb;
 225     }
 226     // Follow chain of load barrier through Similar edges
 227     while (!lb->in(Similar)->is_top()) {
 228       lb = lb->in(Similar)->in(0)->as_LoadBarrier();
 229       assert(lb->in(Address) == in(Address), "");
 230     }
 231     if (lb != in(Similar)->in(0)) {
 232       return lb;
 233     }
 234   }
 235   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
 236     Node* u = val->fast_out(i);
 237     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {
 238       Node* this_ctrl = in(LoadBarrierNode::Control);
 239       Node* other_ctrl = u->in(LoadBarrierNode::Control);
 240       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
 241         return u->as_LoadBarrier();
 242       }
 243     }
 244   }
 245 
 246   if (can_be_eliminated()) {
 247     return NULL;
 248   }
 249 
 250   if (!look_for_similar) {
 251     return NULL;
 252   }
 253 
 254   Node* addr = in(LoadBarrierNode::Address);
 255   for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
 256     Node* u = addr->fast_out(i);
 257     if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
 258       Node* this_ctrl = in(LoadBarrierNode::Control);
 259       Node* other_ctrl = u->in(LoadBarrierNode::Control);
 260       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
 261         ResourceMark rm;
 262         Unique_Node_List wq;
 263         wq.push(in(LoadBarrierNode::Control));
 264         bool ok = true;
 265         bool dom_found = false;
 266         for (uint next = 0; next < wq.size(); ++next) {


 276           if (n == u) {
 277             dom_found = true;
 278             continue;
 279           }
 280           if (n->is_Region()) {
 281             for (uint i = 1; i < n->req(); i++) {
 282               Node* m = n->in(i);
 283               if (m != NULL) {
 284                 wq.push(m);
 285               }
 286             }
 287           } else {
 288             Node* m = n->in(0);
 289             if (m != NULL) {
 290               wq.push(m);
 291             }
 292           }
 293         }
 294         if (ok) {
 295           assert(dom_found, "");
 296           return u->as_LoadBarrier();
 297         }
 298         break;
 299       }
 300     }
 301   }
 302 
 303   return NULL;
 304 }
 305 
 306 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
 307   // Change to that barrier may affect a dominated barrier so re-push those
 308   assert(!is_weak(), "sanity");
 309   Node* val = in(LoadBarrierNode::Oop);
 310 
 311   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
 312     Node* u = val->fast_out(i);
 313     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
 314       Node* this_ctrl = in(Control);
 315       Node* other_ctrl = u->in(Control);
 316       if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
 317         igvn->_worklist.push(u);
 318       }
 319     }
 320 
 321     Node* addr = in(LoadBarrierNode::Address);
 322     for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
 323       Node* u = addr->fast_out(i);
 324       if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {
 325         Node* this_ctrl = in(Control);
 326         Node* other_ctrl = u->in(Control);
 327         if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
 328           igvn->_worklist.push(u);
 329         }
 330       }
 331     }
 332   }
 333 }
 334 
 335 Node *LoadBarrierNode::Identity(PhaseGVN *phase) {





 336   LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
 337   if (dominating_barrier != NULL) {
 338     assert(!is_weak(), "Weak barriers cant be eliminated");
 339     assert(dominating_barrier->in(Oop) == in(Oop), "");
 340     return dominating_barrier;
 341   }
 342 
 343   return this;
 344 }
 345 
 346 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
 347   if (remove_dead_region(phase, can_reshape)) {
 348     return this;
 349   }
 350 
 351   Node *val = in(Oop);
 352   Node *mem = in(Memory);
 353   Node *ctrl = in(Control);
 354 
 355   assert(val->Opcode() != Op_LoadN, "");
 356   assert(val->Opcode() != Op_DecodeN, "");
 357 
 358   if (mem->is_MergeMem()) {
 359     Node *new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
 360     set_req(Memory, new_mem);
 361     if (mem->outcnt() == 0 && can_reshape) {
 362       phase->is_IterGVN()->_worklist.push(mem);
 363     }

 364     return this;
 365   }
 366 
 367   LoadBarrierNode *dominating_barrier = NULL;
 368   if (!is_weak()) {
 369     dominating_barrier = has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress());
 370     if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
 371       assert(in(Address) == dominating_barrier->in(Address), "");
 372       set_req(Similar, dominating_barrier->proj_out(Oop));
 373       return this;
 374     }
 375   }
 376 
 377   bool eliminate = can_reshape && (dominating_barrier != NULL || !has_true_uses());


 378   if (eliminate) {
 379     if (can_reshape) {
 380       PhaseIterGVN* igvn = phase->is_IterGVN();
 381       Node* out_ctrl = proj_out_or_null(Control);
 382       Node* out_res = proj_out_or_null(Oop);
 383 
 384       if (out_ctrl != NULL) {
 385         igvn->replace_node(out_ctrl, ctrl);
 386       }
 387 
 388       // That transformation may cause the Similar edge on the load barrier to be invalid
 389       fix_similar_in_uses(igvn);
 390       if (out_res != NULL) {
 391         if (dominating_barrier != NULL) {
 392           assert(!is_weak(), "Sanity");
 393           igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
 394         } else {
 395           igvn->replace_node(out_res, val);
 396         }
 397       }
 398     }

 399     return new ConINode(TypeInt::ZERO);
 400   }
 401 
 402   // If the Similar edge is no longer a load barrier, clear it
 403   Node* similar = in(Similar);
 404   if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
 405     set_req(Similar, phase->C->top());
 406     return this;
 407   }
 408 
 409   if (can_reshape && !is_weak()) {
 410     // If this barrier is linked through the Similar edge by a
 411     // dominated barrier and both barriers have the same Oop field,
 412     // the dominated barrier can go away, so push it for reprocessing.
 413     // We also want to avoid a barrier to depend on another dominating
 414     // barrier through its Similar edge that itself depend on another
 415     // barrier through its Similar edge and rather have the first
 416     // depend on the third.
 417     PhaseIterGVN* igvn = phase->is_IterGVN();
 418     Node* out_res = proj_out(Oop);
 419     for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 420       Node* u = out_res->fast_out(i);
 421       if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
 422           (u->in(Oop) == val || !u->in(Similar)->is_top())) {
 423         assert(!u->as_LoadBarrier()->is_weak(), "Sanity");
 424         igvn->_worklist.push(u);
 425       }
 426     }

 427     push_dominated_barriers(igvn);
 428   }
 429 
 430   return NULL;
 431 }
 432 
 433 uint LoadBarrierNode::match_edge(uint idx) const {
 434   ShouldNotReachHere();
 435   return 0;
 436 }
 437 
 438 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {
 439   Node* out_res = proj_out_or_null(Oop);
 440   if (out_res == NULL) {
 441     return;
 442   }
 443 
 444   for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 445     Node* u = out_res->fast_out(i);
 446     if (u->is_LoadBarrier() && u->in(Similar) == out_res) {
 447       igvn->replace_input_of(u, Similar, igvn->C->top());
 448       --i;
 449       --imax;
 450     }
 451   }
 452 }
 453 
 454 bool LoadBarrierNode::has_true_uses() const {
 455   Node* out_res = proj_out_or_null(Oop);
 456   if (out_res != NULL) {
 457     for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 458       Node *u = out_res->fast_out(i);
 459       if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
 460         return true;
 461       }


 462     }
 463   }

 464   return false;
 465 }
 466 































































































































































































 467 static bool barrier_needed(C2Access& access) {
 468   return ZBarrierSet::barrier_needed(access.decorators(), access.type());
 469 }
 470 
 471 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
 472   Node* p = BarrierSetC2::load_at_resolved(access, val_type);
 473   if (!barrier_needed(access)) {
 474     return p;
 475   }
 476 
 477   bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
 478   if (p->isa_Load()) {
 479     load_set_barrier(p->as_Load(), weak);
































 480   }
 481   return p;
 482 }
 483 
 484 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 485                                                     Node* new_val, const Type* val_type) const {
 486   Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
 487   LoadStoreNode* lsn = result->as_LoadStore();
 488   if (barrier_needed(access)) {
 489     lsn->set_has_barrier();
 490   }
 491   return lsn;


 492 }
 493 
 494 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 495                                                      Node* new_val, const Type* value_type) const {
 496   Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
 497   LoadStoreNode* lsn = result->as_LoadStore();
 498   if (barrier_needed(access)) {
 499     lsn->set_has_barrier();













 500   }
 501   return lsn;

 502 }
 503 
 504 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
 505   Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
 506   LoadStoreNode* lsn = result->as_LoadStore();
 507   if (barrier_needed(access)) {
 508     lsn->set_has_barrier();
 509   }
 510   return lsn;






 511 }
 512 
 513 // == Macro Expansion ==
 514 
 515 // Optimized, low spill, loadbarrier variant using stub specialized on register used
 516 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {








 517   PhaseIterGVN &igvn = phase->igvn();


































































 518   float unlikely  = PROB_UNLIKELY(0.999);










































































 519 
 520   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
 521   Node* in_mem = barrier->in(LoadBarrierNode::Memory);
 522   Node* in_val = barrier->in(LoadBarrierNode::Oop);
 523   Node* in_adr = barrier->in(LoadBarrierNode::Address);
 524 
 525   Node* out_ctrl = barrier->proj_out_or_null(LoadBarrierNode::Control);
 526   Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
 527 
 528   assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
 529 










 530   Node* jthread = igvn.transform(new ThreadLocalNode());
 531   Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
 532   Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
 533                                                  TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
 534                                                  MemNode::unordered));
 535   Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
 536   Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
 537   Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
 538   Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
 539   IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
 540   Node* then = igvn.transform(new IfTrueNode(iff));
 541   Node* elsen = igvn.transform(new IfFalseNode(iff));
 542 
 543   Node* new_loadp = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
 544                                                                     (const TypePtr*) in_val->bottom_type(), MemNode::unordered, barrier->is_weak()));






 545 


 546   // Create the final region/phi pair to converge cntl/data paths to downstream code
 547   Node* result_region = igvn.transform(new RegionNode(3));
 548   result_region->set_req(1, then);
 549   result_region->set_req(2, elsen);
 550 
 551   Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
 552   result_phi->set_req(1, new_loadp);
 553   result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
 554 
 555   if (out_ctrl != NULL) {


 556     igvn.replace_node(out_ctrl, result_region);
 557   }
 558   igvn.replace_node(out_res, result_phi);
 559 
 560   assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
 561 
 562   igvn.remove_dead_node(barrier);
 563   igvn.remove_dead_node(out_ctrl);
 564   igvn.remove_dead_node(out_res);







 565 
 566   assert(is_gc_barrier_node(result_phi), "sanity");
 567   assert(step_over_gc_barrier(result_phi) == in_val, "sanity");
 568 
 569   phase->C->print_method(PHASE_BARRIER_EXPANSION, 4, barrier->_idx);
 570 }
 571 
 572 bool ZBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
 573   ZBarrierSetC2State* s = state();
 574   if (s->load_barrier_count() > 0) {
 575     PhaseMacroExpand macro(igvn);
 576 


 577     int skipped = 0;
 578     while (s->load_barrier_count() > skipped) {
 579       int load_barrier_count = s->load_barrier_count();
 580       LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
 581       if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
 582         // Node is unreachable, so don't try to expand it
 583         s->remove_load_barrier_node(n);
 584         continue;
 585       }
 586       if (!n->can_be_eliminated()) {
 587         skipped++;
 588         continue;
 589       }
 590       expand_loadbarrier_node(&macro, n);
 591       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
 592       if (C->failing()) {
 593         return true;
 594       }
 595     }
 596     while (s->load_barrier_count() > 0) {
 597       int load_barrier_count = s->load_barrier_count();
 598       LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);
 599       assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");
 600       assert(!n->can_be_eliminated(), "should have been processed already");
 601       expand_loadbarrier_node(&macro, n);
 602       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
 603       if (C->failing()) {
 604         return true;
 605       }
 606     }
 607     igvn.set_delay_transform(false);
 608     igvn.optimize();
 609     if (C->failing()) {
 610       return true;
 611     }
 612   }
 613 
 614   return false;
 615 }
 616 

















































































































































































































































































































































































 617 Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const {
 618   Node* node = c;
 619 
 620   // 1. This step follows potential oop projections of a load barrier before expansion
 621   if (node->is_Proj()) {
 622     node = node->in(0);
 623   }
 624 
 625   // 2. This step checks for unexpanded load barriers
 626   if (node->is_LoadBarrier()) {
 627     return node->in(LoadBarrierNode::Oop);
 628   }
 629 
 630   // 3. This step checks for the phi corresponding to an optimized load barrier expansion
 631   if (node->is_Phi()) {
 632     PhiNode* phi = node->as_Phi();
 633     Node* n = phi->in(1);
 634     if (n != NULL && n->is_LoadBarrierSlowReg()) {
 635       assert(c == node, "projections from step 1 should only be seen before macro expansion");
 636       return phi->in(2);
 637     }
 638   }
 639 
 640   return c;
 641 }
 642 
 643 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
 644   return type == T_OBJECT || type == T_ARRAY;
 645 }
 646 
 647 bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
 648   switch (opcode) {
 649     case Op_LoadBarrier:
 650       assert(0, "There should be no load barriers left");
 651     case Op_ZGetAndSetP:
 652     case Op_ZCompareAndExchangeP:
 653     case Op_ZCompareAndSwapP:
 654     case Op_ZWeakCompareAndSwapP:
 655     case Op_LoadBarrierSlowReg:
 656 #ifdef ASSERT
 657       if (VerifyOptoOopOffsets) {
 658         MemNode *mem = n->as_Mem();
 659         // Check to see if address types have grounded out somehow.
 660         const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
 661         ciInstanceKlass *k = tp->klass()->as_instance_klass();
 662         bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
 663         assert(!tp || oop_offset_is_sane, "");
 664       }
 665 #endif
 666       return true;
 667     default:
 668       return false;
 669   }



 670 }
 671 
 672 bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
 673   switch(opcode) {
 674     case Op_CallLeaf:
 675       if (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() ||
 676           n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr()) {
 677         mem_op = true;
 678         mem_addr_idx = TypeFunc::Parms + 1;
 679         return true;
 680       }
 681       return false;
 682     default:
 683       return false;
 684   }
 685 }
 686 
 687 bool ZBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const {
 688   switch(opcode) {
 689     case Op_ZCompareAndExchangeP:
 690     case Op_ZCompareAndSwapP:
 691     case Op_ZWeakCompareAndSwapP: {
 692       Node *mem = n->in(MemNode::Address);
 693       Node *keepalive = n->in(5);
 694       Node *pair1 = new BinaryNode(mem, keepalive);
 695 
 696       Node *newval = n->in(MemNode::ValueIn);
 697       Node *oldval = n->in(LoadStoreConditionalNode::ExpectedIn);
 698       Node *pair2 = new BinaryNode(oldval, newval);
 699 
 700       n->set_req(MemNode::Address, pair1);
 701       n->set_req(MemNode::ValueIn, pair2);
 702       n->del_req(5);
 703       n->del_req(LoadStoreConditionalNode::ExpectedIn);
 704       return true;
 705     }
 706     case Op_ZGetAndSetP: {
 707       Node *keepalive = n->in(4);
 708       Node *newval = n->in(MemNode::ValueIn);
 709       Node *pair = new BinaryNode(newval, keepalive);
 710       n->set_req(MemNode::ValueIn, pair);
 711       n->del_req(4);
 712       return true;
 713     }
 714 
 715     default:
 716       return false;
 717   }
 718 }
 719 
 720 // == Verification ==
 721 
 722 #ifdef ASSERT
 723 
 724 static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
 725   if (visited.test_set(n->_idx)) {
 726     return true;
 727   }
 728 
 729   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
 730     Node* u = n->fast_out(i);
 731     if (u->is_LoadBarrier()) {
 732     } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) {
 733       if (!look_for_barrier(u, post_parse, visited)) {
 734         return false;
 735       }
 736     } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) {
 737       if (!look_for_barrier(u, post_parse, visited)) {
 738         return false;
 739       }
 740     } else if (u->Opcode() != Op_SCMemProj) {
 741       tty->print("bad use"); u->dump();
 742       return false;
 743     }
 744   }
 745 
 746   return true;
 747 }
 748 
 749 void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
 750   switch(phase) {
 751     case BarrierSetC2::BeforeOptimize:
 752     case BarrierSetC2::BeforeLateInsertion:
 753       assert(state()->load_barrier_count() == 0, "No barriers inserted yet");
 754       break;
 755     case BarrierSetC2::BeforeMacroExpand:
 756       // Barrier placement should be set by now.
 757       verify_gc_barriers(false /*post_parse*/);
 758       break;
 759     case BarrierSetC2::BeforeCodeGen:
 760       // Barriers has been fully expanded.
 761       assert(state()->load_barrier_count() == 0, "No more macro barriers");
 762       break;
 763     default:
 764       assert(0, "Phase without verification");
 765   }
 766 }
 767 
 768 // post_parse implies that there might be load barriers without uses after parsing
 769 // That only applies when adding barriers at parse time.
 770 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
 771   ZBarrierSetC2State* s = state();
 772   Compile* C = Compile::current();
 773   ResourceMark rm;
 774   VectorSet visited(Thread::current()->resource_area());
 775 
 776   for (int i = 0; i < s->load_barrier_count(); i++) {
 777     LoadBarrierNode* n = s->load_barrier_node(i);
 778 
 779     // The dominating barrier on the same address if it exists and
 780     // this barrier must not be applied on the value from the same
 781     // load otherwise the value is not reloaded before it's used the
 782     // second time.
 783     assert(n->in(LoadBarrierNode::Similar)->is_top() ||
 784            (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
 785             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
 786             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
 787            "broken similar edge");
 788 
 789     assert(n->as_LoadBarrier()->has_true_uses(),
 790            "found unneeded load barrier");
 791 
 792     // Several load barrier nodes chained through their Similar edge
 793     // break the code that remove the barriers in final graph reshape.
 794     assert(n->in(LoadBarrierNode::Similar)->is_top() ||
 795            (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
 796             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),
 797            "chain of Similar load barriers");
 798 
 799     if (!n->in(LoadBarrierNode::Similar)->is_top()) {
 800       ResourceMark rm;
 801       Unique_Node_List wq;
 802       Node* other = n->in(LoadBarrierNode::Similar)->in(0);
 803       wq.push(n);


 804       for (uint next = 0; next < wq.size(); ++next) {
 805         Node *nn = wq.at(next);
 806         assert(nn->is_CFG(), "");
 807         assert(!nn->is_SafePoint(), "");
 808 
 809         if (nn == other) {
 810           continue;
 811         }
 812 
 813         if (nn->is_Region()) {
 814           for (uint i = 1; i < nn->req(); i++) {
 815             Node* m = nn->in(i);
 816             if (m != NULL) {
 817               wq.push(m);
 818             }
 819           }
 820         } else {
 821           Node* m = nn->in(0);
 822           if (m != NULL) {
 823             wq.push(m);
 824           }
 825         }
 826       }
 827     }
 828   }
 829 }
 830 
 831 #endif // end verification code
 832 
 833 static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl);
 834 
 835 // This code is cloning all uses of a load that is between a call and the catch blocks,
 836 // to each use.
 837 
 838 static bool fixup_uses_in_catch(PhaseIdealLoop *phase, Node *start_ctrl, Node *node) {
 839 
 840   if (!phase->has_ctrl(node)) {
 841     // This node is floating - doesn't need to be cloned.
 842     assert(node != start_ctrl, "check");
 843     return false;
 844   }
 845 
 846   Node* ctrl = phase->get_ctrl(node);
 847   if (ctrl != start_ctrl) {
 848     // We are in a successor block - the node is ok.
 849     return false; // Unwind
 850   }
 851 
 852   // Process successor nodes
 853   int outcnt = node->outcnt();
 854   for (int i = 0; i < outcnt; i++) {
 855     Node* n = node->raw_out(0);
 856     assert(!n->is_LoadBarrier(), "Sanity");
 857     // Calling recursively, visiting leafs first
 858     fixup_uses_in_catch(phase, start_ctrl, n);
 859   }
 860 
 861   // Now all successors are outside
 862   // - Clone this node to both successors
 863   int no_succs = node->outcnt();
 864   assert(!node->is_Store(), "Stores not expected here");
 865 
 866   // In some very rare cases a load that doesn't need a barrier will end up here
 867   // Treat it as a LoadP and the insertion of phis will be done correctly.
 868   if (node->is_Load()) {
 869     call_catch_cleanup_one(phase, node->as_Load(), phase->get_ctrl(node));
 870   } else {
 871     for (DUIterator_Fast jmax, i = node->fast_outs(jmax); i < jmax; i++) {
 872       Node* use = node->fast_out(i);
 873       Node* clone = node->clone();
 874       assert(clone->outcnt() == 0, "");
 875 
 876       assert(use->find_edge(node) != -1, "check");
 877       phase->igvn().rehash_node_delayed(use);
 878       use->replace_edge(node, clone);
 879 
 880       Node* new_ctrl;
 881       if (use->is_block_start()) {
 882         new_ctrl = use;
 883       } else if (use->is_CFG()) {
 884         new_ctrl = use->in(0);
 885         assert (new_ctrl != NULL, "");
 886       } else {
 887         new_ctrl = phase->get_ctrl(use);
 888       }
 889 
 890       phase->set_ctrl(clone, new_ctrl);
 891 
 892       if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr("  Clone op %i as %i to control %i", node->_idx, clone->_idx, new_ctrl->_idx);
 893       phase->igvn().register_new_node_with_optimizer(clone);
 894       --i, --jmax;
 895     }
 896     assert(node->outcnt() == 0, "must be empty now");
 897 
 898     // Node node is dead.
 899     phase->igvn().remove_dead_node(node);
 900   }
 901   return true; // unwind - return if a use was processed
 902 }
 903 
 904 // Clone a load to a specific catch_proj
 905 static Node* clone_load_to_catchproj(PhaseIdealLoop* phase, Node* load, Node* catch_proj) {
 906   Node* cloned_load = load->clone();
 907   cloned_load->set_req(0, catch_proj);      // set explicit control
 908   phase->set_ctrl(cloned_load, catch_proj); // update
 909   if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr("  Clone LOAD %i as %i to control %i", load->_idx, cloned_load->_idx, catch_proj->_idx);
 910   phase->igvn().register_new_node_with_optimizer(cloned_load);
 911   return cloned_load;
 912 }
 913 
 914 static Node* get_dominating_region(PhaseIdealLoop* phase, Node* node, Node* stop) {
 915   Node* region = node;
 916   while (!region->isa_Region()) {
 917     Node *up = phase->idom(region);
 918     assert(up != region, "Must not loop");
 919     assert(up != stop,   "Must not find original control");
 920     region = up;
 921   }
 922   return region;
 923 }
 924 
 925 // Clone this load to each catch block
 926 static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) {
 927   bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
 928   phase->igvn().set_delay_transform(true);
 929 
 930   // Verify pre conditions
 931   assert(ctrl->isa_Proj() && ctrl->in(0)->isa_Call(), "Must be a call proj");
 932   assert(ctrl->raw_out(0)->isa_Catch(), "Must be a catch");
 933 
 934   if (ctrl->raw_out(0)->isa_Catch()->outcnt() == 1) {
 935     if (trace) tty->print_cr("Cleaning up catch: Skipping load %i, call with single catch", load->_idx);
 936     return;
 937   }
 938 
 939   // Process the loads successor nodes - if any is between
 940   // the call and the catch blocks, they need to be cloned to.
 941   // This is done recursively
 942   int outcnt = load->outcnt();
 943   uint index = 0;
 944   for (int i = 0; i < outcnt; i++) {
 945     if (index < load->outcnt()) {
 946       Node *n = load->raw_out(index);
 947       assert(!n->is_LoadBarrier(), "Sanity");
 948       if (!fixup_uses_in_catch(phase, ctrl, n)) {
 949         // if no successor was cloned, progress to next out.
 950         index++;
 951       }
 952     }
 953   }
 954 
 955   // Now all the loads uses has been cloned down
 956   // Only thing left is to clone the loads, but they must end up
 957   // first in the catch blocks.
 958 
 959   // We clone the loads oo the catch blocks only when needed.
 960   // An array is used to map the catch blocks to each lazily cloned load.
 961   // In that way no extra unnecessary loads are cloned.
 962 
 963   // Any use dominated by original block must have an phi and a region added
 964 
 965   Node* catch_node = ctrl->raw_out(0);
 966   int number_of_catch_projs = catch_node->outcnt();
 967   Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs);
 968   Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs);
 969 
 970   // The phi_map is used to keep track of where phis have already been inserted
 971   int phi_map_len = phase->C->unique();
 972   Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len);
 973   Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len);
 974 
 975   for (unsigned int i = 0; i  < load->outcnt(); i++) {
 976     Node* load_use_control = NULL;
 977     Node* load_use = load->raw_out(i);
 978 
 979     if (phase->has_ctrl(load_use)) {
 980       load_use_control = phase->get_ctrl(load_use);
 981     } else {
 982       load_use_control = load_use->in(0);
 983     }
 984     assert(load_use_control != NULL, "sanity");
 985     if (trace) tty->print_cr("  Handling use: %i, with control: %i", load_use->_idx, load_use_control->_idx);
 986 
 987     // Some times the loads use is a phi. For them we need to determine from which catch block
 988     // the use is defined.
 989     bool load_use_is_phi = false;
 990     unsigned int load_use_phi_index = 0;
 991     Node* phi_ctrl = NULL;
 992     if (load_use->is_Phi()) {
 993       // Find phi input that matches load
 994       for (unsigned int u = 1; u < load_use->req(); u++) {
 995         if (load_use->in(u) == load) {
 996           load_use_is_phi = true;
 997           load_use_phi_index = u;
 998           assert(load_use->in(0)->is_Region(), "Region or broken");
 999           phi_ctrl = load_use->in(0)->in(u);
1000           assert(phi_ctrl->is_CFG(), "check");
1001           assert(phi_ctrl != load,   "check");
1002           break;
1003         }
1004       }
1005       assert(load_use_is_phi,        "must find");
1006       assert(load_use_phi_index > 0, "sanity");
1007     }
1008 
1009     // For each load use, see which catch projs dominates, create load clone lazily and reconnect
1010     bool found_dominating_catchproj = false;
1011     for (int c = 0; c < number_of_catch_projs; c++) {
1012       Node* catchproj = catch_node->raw_out(c);
1013       assert(catchproj != NULL && catchproj->isa_CatchProj(), "Sanity");
1014 
1015       if (!phase->is_dominator(catchproj, load_use_control)) {
1016         if (load_use_is_phi && phase->is_dominator(catchproj, phi_ctrl)) {
1017           // The loads use is local to the catchproj.
1018           // fall out and replace load with catch-local load clone.
1019         } else {
1020           continue;
1021         }
1022       }
1023       assert(!found_dominating_catchproj, "Max one should match");
1024 
1025       // Clone loads to catch projs
1026       Node* load_clone = proj_to_load_mapping[c];
1027       if (load_clone == NULL) {
1028         load_clone = clone_load_to_catchproj(phase, load, catchproj);
1029         proj_to_load_mapping[c] = load_clone;
1030       }
1031       phase->igvn().rehash_node_delayed(load_use);
1032 
1033       if (load_use_is_phi) {
1034         // phis are special - the load is defined from a specific control flow
1035         load_use->set_req(load_use_phi_index, load_clone);
1036       } else {
1037         // Multipe edges can be replaced at once - on calls for example
1038         load_use->replace_edge(load, load_clone);
1039       }
1040       --i; // more than one edge can have been removed, but the next is in later iterations
1041 
1042       // We could break the for-loop after finding a dominating match.
1043       // But keep iterating to catch any bad idom early.
1044       found_dominating_catchproj = true;
1045     }
1046 
1047     // We found no single catchproj that dominated the use - The use is at a point after
1048     // where control flow from multiple catch projs have merged. We will have to create
1049     // phi nodes before the use and tie the output from the cloned loads together. It
1050     // can be a single phi or a number of chained phis, depending on control flow
1051     if (!found_dominating_catchproj) {
1052 
1053       // Use phi-control if use is a phi
1054       if (load_use_is_phi) {
1055         load_use_control = phi_ctrl;
1056       }
1057       assert(phase->is_dominator(ctrl, load_use_control), "Common use but no dominator");
1058 
1059       // Clone a load on all paths
1060       for (int c = 0; c < number_of_catch_projs; c++) {
1061         Node* catchproj = catch_node->raw_out(c);
1062         Node* load_clone = proj_to_load_mapping[c];
1063         if (load_clone == NULL) {
1064           load_clone = clone_load_to_catchproj(phase, load, catchproj);
1065           proj_to_load_mapping[c] = load_clone;
1066         }
1067       }
1068 
1069       // Move up dominator tree from use until dom front is reached
1070       Node* next_region = get_dominating_region(phase, load_use_control, ctrl);
1071       while (phase->idom(next_region) != catch_node) {
1072         next_region = phase->idom(next_region);
1073         if (trace) tty->print_cr("Moving up idom to region ctrl %i", next_region->_idx);
1074       }
1075       assert(phase->is_dominator(catch_node, next_region), "Sanity");
1076 
1077       // Create or reuse phi node that collect all cloned loads and feed it to the use.
1078       Node* test_phi = phi_map[next_region->_idx];
1079       if ((test_phi != NULL) && test_phi->is_Phi()) {
1080         // Reuse an already created phi
1081         if (trace) tty->print_cr("    Using cached Phi %i on load_use %i", test_phi->_idx, load_use->_idx);
1082         phase->igvn().rehash_node_delayed(load_use);
1083         load_use->replace_edge(load, test_phi);
1084         // Now this use is done
1085       } else {
1086         // Otherwise we need to create one or more phis
1087         PhiNode* next_phi = new PhiNode(next_region, load->type());
1088         phi_map[next_region->_idx] = next_phi; // cache new phi
1089         phase->igvn().rehash_node_delayed(load_use);
1090         load_use->replace_edge(load, next_phi);
1091 
1092         int dominators_of_region = 0;
1093         do {
1094           // New phi, connect to region and add all loads as in.
1095           Node* region = next_region;
1096           assert(region->isa_Region() && region->req() > 2, "Catch dead region nodes");
1097           PhiNode* new_phi = next_phi;
1098 
1099           if (trace) tty->print_cr("Created Phi %i on load %i with control %i", new_phi->_idx, load->_idx, region->_idx);
1100 
1101           // Need to add all cloned loads to the phi, taking care that the right path is matched
1102           dominators_of_region = 0; // reset for new region
1103           for (unsigned int reg_i = 1; reg_i < region->req(); reg_i++) {
1104             Node* region_pred = region->in(reg_i);
1105             assert(region_pred->is_CFG(), "check");
1106             bool pred_has_dominator = false;
1107             for (int c = 0; c < number_of_catch_projs; c++) {
1108               Node* catchproj = catch_node->raw_out(c);
1109               if (phase->is_dominator(catchproj, region_pred)) {
1110                 new_phi->set_req(reg_i, proj_to_load_mapping[c]);
1111                 if (trace) tty->print_cr(" - Phi in(%i) set to load %i", reg_i, proj_to_load_mapping[c]->_idx);
1112                 pred_has_dominator = true;
1113                 dominators_of_region++;
1114                 break;
1115               }
1116             }
1117 
1118             // Sometimes we need to chain several phis.
1119             if (!pred_has_dominator) {
1120               assert(dominators_of_region <= 1, "More than one region can't require extra phi");
1121               if (trace) tty->print_cr(" - Region %i pred %i not dominated by catch proj", region->_idx, region_pred->_idx);
1122               // Continue search on on this region_pred
1123               // - walk up to next region
1124               // - create a new phi and connect to first new_phi
1125               next_region = get_dominating_region(phase, region_pred, ctrl);
1126 
1127               // Lookup if there already is a phi, create a new otherwise
1128               Node* test_phi = phi_map[next_region->_idx];
1129               if ((test_phi != NULL) && test_phi->is_Phi()) {
1130                 next_phi = test_phi->isa_Phi();
1131                 dominators_of_region++; // record that a match was found and that we are done
1132                 if (trace) tty->print_cr("    Using cached phi Phi %i on control %i", next_phi->_idx, next_region->_idx);
1133               } else {
1134                 next_phi = new PhiNode(next_region, load->type());
1135                 phi_map[next_region->_idx] = next_phi;
1136               }
1137               new_phi->set_req(reg_i, next_phi);
1138             }
1139           }
1140 
1141           new_phi->set_req(0, region);
1142           phase->igvn().register_new_node_with_optimizer(new_phi);
1143           phase->set_ctrl(new_phi, region);
1144 
1145           assert(dominators_of_region != 0, "Must have found one this iteration");
1146         } while (dominators_of_region == 1);
1147       }
1148       --i;
1149     }
1150   } // end of loop over uses
1151 
1152   assert(load->outcnt() == 0, "All uses should be handled");
1153   phase->igvn().remove_dead_node(load);
1154   phase->C->print_method(PHASE_CALL_CATCH_CLEANUP, 4, load->_idx);
1155 
1156   // Now we should be home
1157   phase->igvn().set_delay_transform(false);
1158 }
1159 
1160 // Sort out the loads that are between a call ant its catch blocks
1161 static void process_catch_cleanup_candidate(PhaseIdealLoop* phase, LoadNode* load) {
1162   bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
1163 
1164   Node* ctrl = phase->get_ctrl(load);
1165   if (!ctrl->is_Proj() || (ctrl->in(0) == NULL) || !ctrl->in(0)->isa_Call()) {
1166     return;
1167   }
1168 
1169   Node* catch_node = ctrl->isa_Proj()->raw_out(0);
1170   if (catch_node->is_Catch()) {
1171     if (catch_node->outcnt() > 1) {
1172       call_catch_cleanup_one(phase, load, ctrl);
1173     } else {
1174       if (trace) tty->print_cr("Call catch cleanup with only one catch: load %i ", load->_idx);
1175     }
1176   }
1177 }
1178 
1179 void ZBarrierSetC2::barrier_insertion_phase(Compile* C, PhaseIterGVN& igvn) const {
1180   PhaseIdealLoop::optimize(igvn, LoopOptsZBarrierInsertion);
1181   if (C->failing())  return;
1182 }
1183 
1184 bool ZBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
1185 
1186   if (mode == LoopOptsZBarrierInsertion) {
1187     // First make sure all loads between call and catch are moved to the catch block
1188     clean_catch_blocks(phase);
1189 
1190     // Then expand barriers on all loads
1191     insert_load_barriers(phase);
1192 
1193     // Handle all Unsafe that need barriers.
1194     insert_barriers_on_unsafe(phase);
1195 
1196     phase->C->clear_major_progress();
1197     return true;
1198   } else {
1199     return false;
1200   }
1201 }
1202 
1203 static bool can_simplify_cas(LoadStoreNode* node) {
1204   if (node->isa_LoadStoreConditional()) {
1205     Node *expected_in = node->as_LoadStoreConditional()->in(LoadStoreConditionalNode::ExpectedIn);
1206     return (expected_in->get_ptr_type() == TypePtr::NULL_PTR);
1207   } else {
1208     return false;
1209   }
1210 }
1211 
1212 static void insert_barrier_before_unsafe(PhaseIdealLoop* phase, LoadStoreNode* old_node) {
1213 
1214   Compile *C = phase->C;
1215   PhaseIterGVN &igvn = phase->igvn();
1216   LoadStoreNode* zclone = NULL;
1217   bool is_weak = false;
1218 
1219   Node *in_ctrl = old_node->in(MemNode::Control);
1220   Node *in_mem  = old_node->in(MemNode::Memory);
1221   Node *in_adr  = old_node->in(MemNode::Address);
1222   Node *in_val  = old_node->in(MemNode::ValueIn);
1223   const TypePtr *adr_type = old_node->adr_type();
1224   const TypePtr* load_type = TypeOopPtr::BOTTOM; // The type for the load we are adding
1225 
1226   switch (old_node->Opcode()) {
1227     case Op_CompareAndExchangeP: {
1228       zclone = new ZCompareAndExchangePNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
1229               adr_type, old_node->get_ptr_type(), ((CompareAndExchangeNode*)old_node)->order());
1230       load_type = old_node->bottom_type()->is_ptr();
1231       break;
1232     }
1233     case Op_WeakCompareAndSwapP: {
1234       if (can_simplify_cas(old_node)) {
1235         break;
1236       }
1237       is_weak  = true;
1238       zclone = new ZWeakCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
1239               ((CompareAndSwapNode*)old_node)->order());
1240       adr_type = TypePtr::BOTTOM;
1241       break;
1242     }
1243     case Op_CompareAndSwapP: {
1244       if (can_simplify_cas(old_node)) {
1245         break;
1246       }
1247       zclone = new ZCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
1248               ((CompareAndSwapNode*)old_node)->order());
1249       adr_type = TypePtr::BOTTOM;
1250       break;
1251     }
1252     case Op_GetAndSetP: {
1253       zclone = new ZGetAndSetPNode(in_ctrl, in_mem, in_adr, in_val, old_node->adr_type(), old_node->get_ptr_type());
1254       load_type = old_node->bottom_type()->is_ptr();
1255       break;
1256     }
1257   }
1258   if (zclone != NULL) {
1259     igvn.register_new_node_with_optimizer(zclone, old_node);
1260 
1261     // Make load
1262     LoadPNode *load = new LoadPNode(NULL, in_mem, in_adr, adr_type, load_type, MemNode::unordered,
1263                                     LoadNode::DependsOnlyOnTest);
1264     load_set_expanded_barrier(load);
1265     igvn.register_new_node_with_optimizer(load);
1266     igvn.replace_node(old_node, zclone);
1267 
1268     Node *barrier = new LoadBarrierNode(C, NULL, in_mem, load, in_adr, is_weak);
1269     Node *barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop);
1270     Node *barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control);
1271 
1272     igvn.register_new_node_with_optimizer(barrier);
1273     igvn.register_new_node_with_optimizer(barrier_val);
1274     igvn.register_new_node_with_optimizer(barrier_ctrl);
1275 
1276     // loop over all of in_ctrl usages and move to barrier_ctrl
1277     for (DUIterator_Last imin, i = in_ctrl->last_outs(imin); i >= imin; --i) {
1278       Node *use = in_ctrl->last_out(i);
1279       uint l;
1280       for (l = 0; use->in(l) != in_ctrl; l++) {}
1281       igvn.replace_input_of(use, l, barrier_ctrl);
1282     }
1283 
1284     load->set_req(MemNode::Control, in_ctrl);
1285     barrier->set_req(LoadBarrierNode::Control, in_ctrl);
1286     zclone->add_req(barrier_val); // add req as keep alive.
1287 
1288     C->print_method(PHASE_ADD_UNSAFE_BARRIER, 4, zclone->_idx);
1289   }
1290 }
1291 
1292 void ZBarrierSetC2::insert_barriers_on_unsafe(PhaseIdealLoop* phase) const {
1293   Compile *C = phase->C;
1294   PhaseIterGVN &igvn = phase->igvn();
1295   uint new_ids = C->unique();
1296   VectorSet visited(Thread::current()->resource_area());
1297   GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
1298   nodeStack.push(C->root());
1299   visited.test_set(C->root()->_idx);
1300 
1301   // Traverse all nodes, visit all unsafe ops that require a barrier
1302   while (nodeStack.length() > 0) {
1303     Node *n = nodeStack.pop();
1304 
1305     bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup
1306     if (is_old_node) {
1307       if (n->is_LoadStore()) {
1308         LoadStoreNode* lsn = n->as_LoadStore();
1309         if (lsn->has_barrier()) {
1310           BasicType bt = lsn->in(MemNode::Address)->bottom_type()->basic_type();
1311           assert ((bt == T_OBJECT || bt == T_ARRAY), "Sanity test");
1312           insert_barrier_before_unsafe(phase, lsn);
1313         }
1314       }
1315     }
1316     for (uint i = 0; i < n->len(); i++) {
1317       if (n->in(i)) {
1318         if (!visited.test_set(n->in(i)->_idx)) {
1319           nodeStack.push(n->in(i));
1320         }
1321       }
1322     }
1323   }
1324 
1325   igvn.optimize();
1326   C->print_method(PHASE_ADD_UNSAFE_BARRIER, 2);
1327 }
1328 
1329 // The purpose of ZBarrierSetC2::clean_catch_blocks is to prepare the IR for
1330 // splicing in load barrier nodes.
1331 //
1332 // The problem is that we might have instructions between a call and its catch nodes.
1333 // (This is usually handled in PhaseCFG:call_catch_cleanup, which clones mach nodes in
1334 // already scheduled blocks.) We can't have loads that require barriers there,
1335 // because we need to splice in new control flow, and that would violate the IR.
1336 //
1337 // clean_catch_blocks find all Loads that require a barrier and clone them and any
1338 // dependent instructions to each use. The loads must be in the beginning of the catch block
1339 // before any store.
1340 //
1341 // Sometimes the loads use will be at a place dominated by all catch blocks, then we need
1342 // a load in each catch block, and a Phi at the dominated use.
1343 
1344 void ZBarrierSetC2::clean_catch_blocks(PhaseIdealLoop* phase) const {
1345 
1346   Compile *C = phase->C;
1347   uint new_ids = C->unique();
1348   PhaseIterGVN &igvn = phase->igvn();
1349   VectorSet visited(Thread::current()->resource_area());
1350   GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
1351   nodeStack.push(C->root());
1352   visited.test_set(C->root()->_idx);
1353 
1354   // Traverse all nodes, visit all loads that require a barrier
1355   while(nodeStack.length() > 0) {
1356     Node *n = nodeStack.pop();
1357 
1358     for (uint i = 0; i < n->len(); i++) {
1359       if (n->in(i)) {
1360         if (!visited.test_set(n->in(i)->_idx)) {
1361           nodeStack.push(n->in(i));
1362         }
1363       }
1364     }
1365 
1366     bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup
1367     if (n->is_Load() && is_old_node) {
1368       LoadNode* load = n->isa_Load();
1369       // only care about loads that will have a barrier
1370       if (load_require_barrier(load)) {
1371         process_catch_cleanup_candidate(phase, load);
1372       }
1373     }
1374   }
1375 
1376   C->print_method(PHASE_CALL_CATCH_CLEANUP, 2);
1377 }
1378 
1379 class DomDepthCompareClosure : public CompareClosure<LoadNode*> {
1380   PhaseIdealLoop* _phase;
1381 
1382 public:
1383   DomDepthCompareClosure(PhaseIdealLoop* phase) : _phase(phase) { }
1384 
1385   int do_compare(LoadNode* const &n1, LoadNode* const &n2) {
1386     int d1 = _phase->dom_depth(_phase->get_ctrl(n1));
1387     int d2 = _phase->dom_depth(_phase->get_ctrl(n2));
1388     if (d1 == d2) {
1389       // Compare index if the depth is the same, ensures all entries are unique.
1390       return n1->_idx - n2->_idx;
1391     } else {
1392       return d2 - d1;
1393     }
1394   }
1395 };
1396 
1397 // Traverse graph and add all loadPs to list, sorted by dom depth
1398 void gather_loadnodes_sorted(PhaseIdealLoop* phase, GrowableArray<LoadNode*>* loadList) {
1399 
1400   VectorSet visited(Thread::current()->resource_area());
1401   GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
1402   DomDepthCompareClosure ddcc(phase);
1403 
1404   nodeStack.push(phase->C->root());
1405   while(nodeStack.length() > 0) {
1406     Node *n = nodeStack.pop();
1407     if (visited.test(n->_idx)) {
1408       continue;
1409     }
1410 
1411     if (n->isa_Load()) {
1412       LoadNode *load = n->as_Load();
1413       if (load_require_barrier(load)) {
1414         assert(phase->get_ctrl(load) != NULL, "sanity");
1415         assert(phase->dom_depth(phase->get_ctrl(load)) != 0, "sanity");
1416         loadList->insert_sorted(&ddcc, load);
1417       }
1418     }
1419 
1420     visited.set(n->_idx);
1421     for (uint i = 0; i < n->req(); i++) {
1422       if (n->in(i)) {
1423         if (!visited.test(n->in(i)->_idx)) {
1424           nodeStack.push(n->in(i));
1425         }
1426       }
1427     }
1428   }
1429 }
1430 
1431 // Add LoadBarriers to all LoadPs
1432 void ZBarrierSetC2::insert_load_barriers(PhaseIdealLoop* phase) const {
1433 
1434   bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
1435   GrowableArray<LoadNode *> loadList(Thread::current()->resource_area(), 0, 0, NULL);
1436   gather_loadnodes_sorted(phase, &loadList);
1437 
1438   PhaseIterGVN &igvn = phase->igvn();
1439   int count = 0;
1440 
1441   for (GrowableArrayIterator<LoadNode *> loadIter = loadList.begin(); loadIter != loadList.end(); ++loadIter) {
1442     LoadNode *load = *loadIter;
1443 
1444     if (load_has_expanded_barrier(load)) {
1445       continue;
1446     }



1447 
1448     do {
1449       // Insert a barrier on a loadP
1450       // if another load is found that needs to be expanded first, retry on that one
1451       LoadNode* result = insert_one_loadbarrier(phase, load, phase->get_ctrl(load));
1452       while (result != NULL) {
1453         result = insert_one_loadbarrier(phase, result, phase->get_ctrl(result));
1454       }
1455     } while (!load_has_expanded_barrier(load));

1456   }
1457 
1458   phase->C->print_method(PHASE_INSERT_BARRIER, 2);
1459 }
1460 
1461 void push_antidependent_stores(PhaseIdealLoop* phase, Node_Stack& nodestack, LoadNode* start_load) {
1462   // push all stores on the same mem, that can_alias
1463   // Any load found must be handled first
1464   PhaseIterGVN &igvn = phase->igvn();
1465   int load_alias_idx = igvn.C->get_alias_index(start_load->adr_type());
1466 
1467   Node *mem = start_load->in(1);
1468   for (DUIterator_Fast imax, u = mem->fast_outs(imax); u < imax; u++) {
1469     Node *mem_use = mem->fast_out(u);
1470 
1471     if (mem_use == start_load) continue;
1472     if (!mem_use->is_Store()) continue;
1473     if (!phase->has_ctrl(mem_use)) continue;
1474     if (phase->get_ctrl(mem_use) != phase->get_ctrl(start_load)) continue;
1475 
1476     // add any aliasing store in this block
1477     StoreNode *store = mem_use->isa_Store();
1478     const TypePtr *adr_type = store->adr_type();
1479     if (igvn.C->can_alias(adr_type, load_alias_idx)) {
1480       nodestack.push(store, 0);
1481     }
1482   }
1483 }
1484 
1485 LoadNode* ZBarrierSetC2::insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* start_load, Node* ctrl) const {
1486   bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
1487   PhaseIterGVN &igvn = phase->igvn();
1488 
1489   // Check for other loadPs at the same loop depth that is reachable by a DFS
1490   // - if found - return it. It needs to be inserted first
1491   // - otherwise proceed and insert barrier
1492 
1493   VectorSet visited(Thread::current()->resource_area());
1494   Node_Stack nodestack(100);
1495 
1496   nodestack.push(start_load, 0);
1497   push_antidependent_stores(phase, nodestack, start_load);
1498 
1499   while(!nodestack.is_empty()) {
1500     Node* n = nodestack.node(); // peek
1501     nodestack.pop();
1502     if (visited.test(n->_idx)) {
1503       continue;
1504     }
1505 
1506     if (n->is_Load() && n != start_load && load_require_barrier(n->as_Load()) && !load_has_expanded_barrier(n->as_Load())) {
1507       // Found another load that needs a barrier in the same block. Must expand later loads first.
1508       if (trace) tty->print_cr(" * Found LoadP %i on DFS", n->_idx);
1509       return n->as_Load(); // return node that should be expanded first
1510     }
1511 
1512     if (!phase->has_ctrl(n)) continue;
1513     if (phase->get_ctrl(n) != phase->get_ctrl(start_load)) continue;
1514     if (n->is_Phi()) continue;
1515 
1516     visited.set(n->_idx);
1517     // push all children
1518     for (DUIterator_Fast imax, ii = n->fast_outs(imax); ii < imax; ii++) {
1519       Node* c = n->fast_out(ii);
1520       if (c != NULL) {
1521         nodestack.push(c, 0);
1522       }
1523     }
1524   }
1525 
1526   insert_one_loadbarrier_inner(phase, start_load, ctrl, visited);
1527   return NULL;
1528 }
1529 
1530 void ZBarrierSetC2::insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited2) const {
1531   PhaseIterGVN &igvn = phase->igvn();
1532   Compile* C = igvn.C;
1533   bool trace = C->directive()->ZTraceLoadBarriersOption;
1534 
1535   // create barrier
1536   Node* barrier = new LoadBarrierNode(C, NULL, load->in(LoadNode::Memory), NULL, load->in(LoadNode::Address), load_has_weak_barrier(load));
1537   Node* barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop);
1538   Node* barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control);
1539 
1540   if (trace) tty->print_cr("Insert load %i with barrier: %i and ctrl : %i", load->_idx, barrier->_idx, ctrl->_idx);
1541 
1542   // Splice control
1543   // - insert barrier control diamond between loads ctrl and ctrl successor on path to block end.
1544   // - If control successor is a catch, step over to next.
1545   Node* ctrl_succ = NULL;
1546   for (DUIterator_Fast imax, j = ctrl->fast_outs(imax); j < imax; j++) {
1547     Node* tmp = ctrl->fast_out(j);
1548 
1549     // - CFG nodes is the ones we are going to splice (1 only!)
1550     // - Phi nodes will continue to hang from the region node!
1551     // - self loops should be skipped
1552     if (tmp->is_Phi() || tmp == ctrl) {
1553       continue;
1554     }
1555 
1556     if (tmp->is_CFG()) {
1557       assert(ctrl_succ == NULL, "There can be only one");
1558       ctrl_succ = tmp;
1559       continue;
1560     }
1561   }
1562 
1563   // Now splice control
1564   assert(ctrl_succ != load, "sanity");
1565   assert(ctrl_succ != NULL, "Broken IR");
1566   bool found = false;
1567   for(uint k = 0; k < ctrl_succ->req(); k++) {
1568     if (ctrl_succ->in(k) == ctrl) {
1569       assert(!found, "sanity");
1570       if (trace) tty->print_cr(" Move CFG ctrl_succ %i to barrier_ctrl", ctrl_succ->_idx);
1571       igvn.replace_input_of(ctrl_succ, k, barrier_ctrl);
1572       found = true;
1573       k--;
1574     }
1575   }
1576 
1577   // For all successors of ctrl - move all visited to become successors of barrier_ctrl instead
1578   for (DUIterator_Fast imax, r = ctrl->fast_outs(imax); r < imax; r++) {
1579     Node* tmp = ctrl->fast_out(r);
1580     if (visited2.test(tmp->_idx) && (tmp != load)) {
1581       if (trace) tty->print_cr(" Move ctrl_succ %i to barrier_ctrl", tmp->_idx);
1582       igvn.replace_input_of(tmp, 0, barrier_ctrl);
1583       --r; --imax;
1584     }
1585   }
1586 
1587   // Move the loads user to the barrier
1588   for (DUIterator_Fast imax, i = load->fast_outs(imax); i < imax; i++) {
1589     Node* u = load->fast_out(i);
1590     if (u->isa_LoadBarrier()) {
1591       continue;
1592     }
1593 
1594     // find correct input  - replace with iterator?
1595     for(uint j = 0; j < u->req(); j++) {
1596       if (u->in(j) == load) {
1597         igvn.replace_input_of(u, j, barrier_val);
1598         --i; --imax; // Adjust the iterator of the *outer* loop
1599         break; // some nodes (calls) might have several uses from the same node
1600       }
1601     }

1602   }
1603 
1604   // Connect barrier to load and control
1605   barrier->set_req(LoadBarrierNode::Oop, load);
1606   barrier->set_req(LoadBarrierNode::Control, ctrl);
1607 
1608   igvn.rehash_node_delayed(load);
1609   igvn.register_new_node_with_optimizer(barrier);
1610   igvn.register_new_node_with_optimizer(barrier_val);
1611   igvn.register_new_node_with_optimizer(barrier_ctrl);
1612   load_set_expanded_barrier(load);
1613 
1614   C->print_method(PHASE_INSERT_BARRIER, 3, load->_idx);
1615 }
1616 
1617 // The bad_mask in the ThreadLocalData shouldn't have an anti-dep-check.
1618 // The bad_mask address if of type TypeRawPtr, but that will alias
1619 // InitializeNodes until the type system is expanded.
1620 bool ZBarrierSetC2::needs_anti_dependence_check(const Node* node) const {
1621   MachNode* mnode = node->as_Mach();
1622   if (mnode != NULL) {
1623     intptr_t offset = 0;
1624     const TypePtr *adr_type2 = NULL;
1625     const Node* base = mnode->get_base_and_disp(offset, adr_type2);
1626     if ((base != NULL) &&
1627         (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadLocal) &&
1628         (offset == in_bytes(ZThreadLocalData::address_bad_mask_offset()))) {
1629       return false;
1630     }
1631   }
1632   return true;
1633 }


   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"

  25 #include "opto/compile.hpp"
  26 #include "opto/castnode.hpp"
  27 #include "opto/escape.hpp"
  28 #include "opto/graphKit.hpp"
  29 #include "opto/idealKit.hpp"
  30 #include "opto/loopnode.hpp"

  31 #include "opto/macro.hpp"


  32 #include "opto/node.hpp"



  33 #include "opto/type.hpp"


  34 #include "utilities/macros.hpp"
  35 #include "gc/z/zBarrierSet.hpp"
  36 #include "gc/z/c2/zBarrierSetC2.hpp"
  37 #include "gc/z/zThreadLocalData.hpp"
  38 #include "gc/z/zBarrierSetRuntime.hpp"
  39 
  40 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena) :
  41     _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8,  0, NULL)) {}
  42 
  43 int ZBarrierSetC2State::load_barrier_count() const {
  44   return _load_barrier_nodes->length();
  45 }
  46 
  47 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {
  48   assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");
  49   _load_barrier_nodes->append(n);
  50 }
  51 
  52 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {
  53   // this function may be called twice for a node so check


  67 
  68 ZBarrierSetC2State* ZBarrierSetC2::state() const {
  69   return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
  70 }
  71 
  72 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {
  73   // 1. This step follows potential oop projections of a load barrier before expansion
  74   if (node->is_Proj()) {
  75     node = node->in(0);
  76   }
  77 
  78   // 2. This step checks for unexpanded load barriers
  79   if (node->is_LoadBarrier()) {
  80     return true;
  81   }
  82 
  83   // 3. This step checks for the phi corresponding to an optimized load barrier expansion
  84   if (node->is_Phi()) {
  85     PhiNode* phi = node->as_Phi();
  86     Node* n = phi->in(1);
  87     if (n != NULL && (n->is_LoadBarrierSlowReg() ||  n->is_LoadBarrierWeakSlowReg())) {
  88       return true;
  89     }
  90   }
  91 
  92   return false;
  93 }
  94 
  95 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {
  96   if (node->is_LoadBarrier()) {
  97     state()->add_load_barrier_node(node->as_LoadBarrier());
  98   }
  99 }
 100 
 101 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
 102   if (node->is_LoadBarrier()) {
 103     state()->remove_load_barrier_node(node->as_LoadBarrier());
 104   }
 105 }
 106 
 107 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {
 108   // Remove useless LoadBarrier nodes
 109   ZBarrierSetC2State* s = state();
 110   for (int i = s->load_barrier_count()-1; i >= 0; i--) {
 111     LoadBarrierNode* n = s->load_barrier_node(i);
 112     if (!useful.member(n)) {
 113       unregister_potential_barrier_node(n);
 114     }
 115   }
 116 }
 117 
 118 void ZBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
 119   if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
 120     igvn->_worklist.push(node);
 121   }
 122 }
 123 
 124 void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) {
 125   // Look for dominating barriers on the same address only once all
 126   // other loop opts are over. Loop opts may cause a safepoint to be
 127   // inserted between a barrier and its dominating barrier.
 128   Compile* C = Compile::current();
 129   ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2();
 130   ZBarrierSetC2State* s = bs->state();
 131   if (s->load_barrier_count() >= 2) {
 132     Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]);
 133     PhaseIdealLoop::optimize(igvn, LoopOptsLastRound);
 134     if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
 135   }
 136 }
 137 
 138 void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {
 139   // Permanent temporary workaround
 140   // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is
 141   // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to
 142   // the worklist. Unless we add them explicitly they are not guaranteed to end up there.
 143   ZBarrierSetC2State* s = state();
 144 
 145   for (int i = 0; i < s->load_barrier_count(); i++) {
 146     LoadBarrierNode* n = s->load_barrier_node(i);
 147     worklist->push(n);
 148   }
 149 }
 150 
 151 const TypeFunc* ZBarrierSetC2::load_barrier_Type() const {
 152   const Type** fields;
 153 
 154   // Create input types (domain)
 155   fields = TypeTuple::fields(2);
 156   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
 157   fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM;
 158   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
 159 
 160   // Create result type (range)
 161   fields = TypeTuple::fields(1);
 162   fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM;
 163   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
 164 
 165   return TypeFunc::make(domain, range);
 166 }
 167 
 168 // == LoadBarrierNode ==
 169 
 170 LoadBarrierNode::LoadBarrierNode(Compile* C,
 171                                  Node* c,
 172                                  Node* mem,
 173                                  Node* val,
 174                                  Node* adr,
 175                                  bool weak,
 176                                  bool writeback,
 177                                  bool oop_reload_allowed) :
 178     MultiNode(Number_of_Inputs),
 179     _weak(weak),
 180     _writeback(writeback),
 181     _oop_reload_allowed(oop_reload_allowed) {
 182   init_req(Control, c);
 183   init_req(Memory, mem);
 184   init_req(Oop, val);
 185   init_req(Address, adr);
 186   init_req(Similar, C->top());
 187 
 188   init_class_id(Class_LoadBarrier);
 189   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 190   bs->register_potential_barrier_node(this);
 191 }
 192 
 193 uint LoadBarrierNode::size_of() const {
 194   return sizeof(*this);
 195 }
 196 
 197 bool LoadBarrierNode::cmp(const Node& n) const {
 198   ShouldNotReachHere();
 199   return false;
 200 }
 201 
 202 const Type *LoadBarrierNode::bottom_type() const {
 203   const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
 204   Node* in_oop = in(Oop);
 205   floadbarrier[Control] = Type::CONTROL;
 206   floadbarrier[Memory] = Type::MEMORY;
 207   floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();
 208   return TypeTuple::make(Number_of_Outputs, floadbarrier);
 209 }
 210 
 211 const TypePtr* LoadBarrierNode::adr_type() const {
 212   ShouldNotReachHere();
 213   return NULL;
 214 }
 215 
 216 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
 217   const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
 218   const Type* val_t = phase->type(in(Oop));
 219   floadbarrier[Control] = Type::CONTROL;
 220   floadbarrier[Memory] = Type::MEMORY;
 221   floadbarrier[Oop] = val_t;
 222   return TypeTuple::make(Number_of_Outputs, floadbarrier);
 223 }
 224 
 225 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
 226   if (phase != NULL) {
 227     return phase->is_dominator(d, n);
 228   }
 229 
 230   for (int i = 0; i < 10 && n != NULL; i++) {
 231     n = IfNode::up_one_dom(n, linear_only);
 232     if (n == d) {
 233       return true;
 234     }
 235   }
 236 
 237   return false;
 238 }
 239 
 240 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {





 241   Node* val = in(LoadBarrierNode::Oop);
 242   if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
 243     LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
 244     assert(lb->in(Address) == in(Address), "");
 245     // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
 246     if (lb->in(Oop) == in(Oop)) {
 247       return lb;
 248     }
 249     // Follow chain of load barrier through Similar edges
 250     while (!lb->in(Similar)->is_top()) {
 251       lb = lb->in(Similar)->in(0)->as_LoadBarrier();
 252       assert(lb->in(Address) == in(Address), "");
 253     }
 254     if (lb != in(Similar)->in(0)) {
 255       return lb;
 256     }
 257   }
 258   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
 259     Node* u = val->fast_out(i);
 260     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {
 261       Node* this_ctrl = in(LoadBarrierNode::Control);
 262       Node* other_ctrl = u->in(LoadBarrierNode::Control);
 263       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
 264         return u->as_LoadBarrier();
 265       }
 266     }
 267   }
 268 
 269   if (ZVerifyLoadBarriers || can_be_eliminated()) {
 270     return NULL;
 271   }
 272 
 273   if (!look_for_similar) {
 274     return NULL;
 275   }
 276 
 277   Node* addr = in(LoadBarrierNode::Address);
 278   for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
 279     Node* u = addr->fast_out(i);
 280     if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
 281       Node* this_ctrl = in(LoadBarrierNode::Control);
 282       Node* other_ctrl = u->in(LoadBarrierNode::Control);
 283       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
 284         ResourceMark rm;
 285         Unique_Node_List wq;
 286         wq.push(in(LoadBarrierNode::Control));
 287         bool ok = true;
 288         bool dom_found = false;
 289         for (uint next = 0; next < wq.size(); ++next) {


 299           if (n == u) {
 300             dom_found = true;
 301             continue;
 302           }
 303           if (n->is_Region()) {
 304             for (uint i = 1; i < n->req(); i++) {
 305               Node* m = n->in(i);
 306               if (m != NULL) {
 307                 wq.push(m);
 308               }
 309             }
 310           } else {
 311             Node* m = n->in(0);
 312             if (m != NULL) {
 313               wq.push(m);
 314             }
 315           }
 316         }
 317         if (ok) {
 318           assert(dom_found, "");
 319           return u->as_LoadBarrier();;
 320         }
 321         break;
 322       }
 323     }
 324   }
 325 
 326   return NULL;
 327 }
 328 
 329 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
 330   // Change to that barrier may affect a dominated barrier so re-push those

 331   Node* val = in(LoadBarrierNode::Oop);
 332 
 333   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
 334     Node* u = val->fast_out(i);
 335     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
 336       Node* this_ctrl = in(Control);
 337       Node* other_ctrl = u->in(Control);
 338       if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
 339         igvn->_worklist.push(u);
 340       }
 341     }
 342 
 343     Node* addr = in(LoadBarrierNode::Address);
 344     for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
 345       Node* u = addr->fast_out(i);
 346       if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {
 347         Node* this_ctrl = in(Control);
 348         Node* other_ctrl = u->in(Control);
 349         if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
 350           igvn->_worklist.push(u);
 351         }
 352       }
 353     }
 354   }
 355 }
 356 
 357 Node *LoadBarrierNode::Identity(PhaseGVN *phase) {
 358   if (!phase->C->directive()->ZOptimizeLoadBarriersOption) {
 359     return this;
 360   }
 361 
 362   bool redundant_addr = false;
 363   LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
 364   if (dominating_barrier != NULL) {

 365     assert(dominating_barrier->in(Oop) == in(Oop), "");
 366     return dominating_barrier;
 367   }
 368 
 369   return this;
 370 }
 371 
 372 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
 373   if (remove_dead_region(phase, can_reshape)) {
 374     return this;
 375   }
 376 
 377   Node* val = in(Oop);
 378   Node* mem = in(Memory);
 379   Node* ctrl = in(Control);
 380   Node* adr = in(Address);
 381   assert(val->Opcode() != Op_LoadN, "");

 382 
 383   if (mem->is_MergeMem()) {
 384     Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
 385     set_req(Memory, new_mem);
 386     if (mem->outcnt() == 0 && can_reshape) {
 387       phase->is_IterGVN()->_worklist.push(mem);
 388     }
 389 
 390     return this;
 391   }
 392 
 393   bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption;
 394   LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL;
 395   if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
 396     assert(in(Address) == dominating_barrier->in(Address), "");
 397     set_req(Similar, dominating_barrier->proj_out(Oop));
 398     return this;


 399   }
 400 
 401   bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) ||
 402                    (can_reshape && (dominating_barrier != NULL || !has_true_uses()));
 403 
 404   if (eliminate) {
 405     if (can_reshape) {
 406       PhaseIterGVN* igvn = phase->is_IterGVN();
 407       Node* out_ctrl = proj_out_or_null(Control);
 408       Node* out_res = proj_out_or_null(Oop);
 409 
 410       if (out_ctrl != NULL) {
 411         igvn->replace_node(out_ctrl, ctrl);
 412       }
 413 
 414       // That transformation may cause the Similar edge on the load barrier to be invalid
 415       fix_similar_in_uses(igvn);
 416       if (out_res != NULL) {
 417         if (dominating_barrier != NULL) {

 418           igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
 419         } else {
 420           igvn->replace_node(out_res, val);
 421         }
 422       }
 423     }
 424 
 425     return new ConINode(TypeInt::ZERO);
 426   }
 427 
 428   // If the Similar edge is no longer a load barrier, clear it
 429   Node* similar = in(Similar);
 430   if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
 431     set_req(Similar, phase->C->top());
 432     return this;
 433   }
 434 
 435   if (can_reshape) {
 436     // If this barrier is linked through the Similar edge by a
 437     // dominated barrier and both barriers have the same Oop field,
 438     // the dominated barrier can go away, so push it for reprocessing.
 439     // We also want to avoid a barrier to depend on another dominating
 440     // barrier through its Similar edge that itself depend on another
 441     // barrier through its Similar edge and rather have the first
 442     // depend on the third.
 443     PhaseIterGVN* igvn = phase->is_IterGVN();
 444     Node* out_res = proj_out(Oop);
 445     for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 446       Node* u = out_res->fast_out(i);
 447       if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
 448           (u->in(Oop) == val || !u->in(Similar)->is_top())) {

 449         igvn->_worklist.push(u);
 450       }
 451     }
 452 
 453     push_dominated_barriers(igvn);
 454   }
 455 
 456   return NULL;
 457 }
 458 
 459 uint LoadBarrierNode::match_edge(uint idx) const {
 460   ShouldNotReachHere();
 461   return 0;
 462 }
 463 
 464 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {
 465   Node* out_res = proj_out_or_null(Oop);
 466   if (out_res == NULL) {
 467     return;
 468   }
 469 
 470   for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 471     Node* u = out_res->fast_out(i);
 472     if (u->is_LoadBarrier() && u->in(Similar) == out_res) {
 473       igvn->replace_input_of(u, Similar, igvn->C->top());
 474       --i;
 475       --imax;
 476     }
 477   }
 478 }
 479 
 480 bool LoadBarrierNode::has_true_uses() const {
 481   Node* out_res = proj_out_or_null(Oop);
 482   if (out_res == NULL) {
 483     return false;
 484   }
 485 
 486   for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 487     Node* u = out_res->fast_out(i);
 488     if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
 489       return true;
 490     }
 491   }
 492 
 493   return false;
 494 }
 495 
 496 // == Accesses ==
 497 
 498 Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicParseAccess& access) const {
 499   assert(!UseCompressedOops, "Not allowed");
 500   CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access();
 501   PhaseGVN& gvn = access.gvn();
 502   Compile* C = Compile::current();
 503   GraphKit* kit = access.kit();
 504 
 505   Node* in_ctrl     = cas->in(MemNode::Control);
 506   Node* in_mem      = cas->in(MemNode::Memory);
 507   Node* in_adr      = cas->in(MemNode::Address);
 508   Node* in_val      = cas->in(MemNode::ValueIn);
 509   Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn);
 510 
 511   float likely                   = PROB_LIKELY(0.999);
 512 
 513   const TypePtr *adr_type        = gvn.type(in_adr)->isa_ptr();
 514   Compile::AliasType* alias_type = C->alias_type(adr_type);
 515   int alias_idx                  = C->get_alias_index(adr_type);
 516 
 517   // Outer check - true: continue, false: load and check
 518   Node* region   = new RegionNode(3);
 519   Node* phi      = new PhiNode(region, TypeInt::BOOL);
 520   Node* phi_mem  = new PhiNode(region, Type::MEMORY, adr_type);
 521 
 522   // Inner check - is the healed ref equal to the expected
 523   Node* region2  = new RegionNode(3);
 524   Node* phi2     = new PhiNode(region2, TypeInt::BOOL);
 525   Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type);
 526 
 527   // CAS node returns 0 or 1
 528   Node* cmp     = gvn.transform(new CmpINode(cas, kit->intcon(0)));
 529   Node* bol     = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
 530   IfNode* iff   = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
 531   Node* then    = gvn.transform(new IfTrueNode(iff));
 532   Node* elsen   = gvn.transform(new IfFalseNode(iff));
 533 
 534   Node* scmemproj1   = gvn.transform(new SCMemProjNode(cas));
 535 
 536   kit->set_memory(scmemproj1, alias_idx);
 537   phi_mem->init_req(1, scmemproj1);
 538   phi_mem2->init_req(2, scmemproj1);
 539 
 540   // CAS fail - reload and heal oop
 541   Node* reload      = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
 542   Node* barrier     = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
 543   Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
 544   Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
 545 
 546   // Check load
 547   Node* tmpX    = gvn.transform(new CastP2XNode(NULL, barrierdata));
 548   Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
 549   Node* cmp2    = gvn.transform(new CmpXNode(tmpX, in_expX));
 550   Node *bol2    = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
 551   IfNode* iff2  = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
 552   Node* then2   = gvn.transform(new IfTrueNode(iff2));
 553   Node* elsen2  = gvn.transform(new IfFalseNode(iff2));
 554 
 555   // redo CAS
 556   Node* cas2       = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order()));
 557   Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2));
 558   kit->set_control(elsen2);
 559   kit->set_memory(scmemproj2, alias_idx);
 560 
 561   // Merge inner flow - check if healed oop was equal too expected.
 562   region2->set_req(1, kit->control());
 563   region2->set_req(2, then2);
 564   phi2->set_req(1, cas2);
 565   phi2->set_req(2, kit->intcon(0));
 566   phi_mem2->init_req(1, scmemproj2);
 567   kit->set_memory(phi_mem2, alias_idx);
 568 
 569   // Merge outer flow - then check if first CAS succeeded
 570   region->set_req(1, then);
 571   region->set_req(2, region2);
 572   phi->set_req(1, kit->intcon(1));
 573   phi->set_req(2, phi2);
 574   phi_mem->init_req(2, phi_mem2);
 575   kit->set_memory(phi_mem, alias_idx);
 576 
 577   gvn.transform(region2);
 578   gvn.transform(phi2);
 579   gvn.transform(phi_mem2);
 580   gvn.transform(region);
 581   gvn.transform(phi);
 582   gvn.transform(phi_mem);
 583 
 584   kit->set_control(region);
 585   kit->insert_mem_bar(Op_MemBarCPUOrder);
 586 
 587   return phi;
 588 }
 589 
 590 Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicParseAccess& access) const {
 591   CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access();
 592   GraphKit* kit = access.kit();
 593   PhaseGVN& gvn = kit->gvn();
 594   Compile* C = Compile::current();
 595 
 596   Node* in_ctrl     = cmpx->in(MemNode::Control);
 597   Node* in_mem      = cmpx->in(MemNode::Memory);
 598   Node* in_adr      = cmpx->in(MemNode::Address);
 599   Node* in_val      = cmpx->in(MemNode::ValueIn);
 600   Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn);
 601 
 602   float likely                   = PROB_LIKELY(0.999);
 603 
 604   const TypePtr *adr_type        = cmpx->get_ptr_type();
 605   Compile::AliasType* alias_type = C->alias_type(adr_type);
 606   int alias_idx                  = C->get_alias_index(adr_type);
 607 
 608   // Outer check - true: continue, false: load and check
 609   Node* region  = new RegionNode(3);
 610   Node* phi     = new PhiNode(region, adr_type);
 611 
 612   // Inner check - is the healed ref equal to the expected
 613   Node* region2 = new RegionNode(3);
 614   Node* phi2    = new PhiNode(region2, adr_type);
 615 
 616   // Check if cmpx succeeded
 617   Node* cmp     = gvn.transform(new CmpPNode(cmpx, in_expected));
 618   Node* bol     = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool();
 619   IfNode* iff   = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
 620   Node* then    = gvn.transform(new IfTrueNode(iff));
 621   Node* elsen   = gvn.transform(new IfFalseNode(iff));
 622 
 623   Node* scmemproj1  = gvn.transform(new SCMemProjNode(cmpx));
 624   kit->set_memory(scmemproj1, alias_idx);
 625 
 626   // CAS fail - reload and heal oop
 627   Node* reload      = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
 628   Node* barrier     = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
 629   Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
 630   Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
 631 
 632   // Check load
 633   Node* tmpX    = gvn.transform(new CastP2XNode(NULL, barrierdata));
 634   Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
 635   Node* cmp2    = gvn.transform(new CmpXNode(tmpX, in_expX));
 636   Node *bol2    = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
 637   IfNode* iff2  = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
 638   Node* then2   = gvn.transform(new IfTrueNode(iff2));
 639   Node* elsen2  = gvn.transform(new IfFalseNode(iff2));
 640 
 641   // Redo CAS
 642   Node* cmpx2      = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order()));
 643   Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2));
 644   kit->set_control(elsen2);
 645   kit->set_memory(scmemproj2, alias_idx);
 646 
 647   // Merge inner flow - check if healed oop was equal too expected.
 648   region2->set_req(1, kit->control());
 649   region2->set_req(2, then2);
 650   phi2->set_req(1, cmpx2);
 651   phi2->set_req(2, barrierdata);
 652 
 653   // Merge outer flow - then check if first cas succeeded
 654   region->set_req(1, then);
 655   region->set_req(2, region2);
 656   phi->set_req(1, cmpx);
 657   phi->set_req(2, phi2);
 658 
 659   gvn.transform(region2);
 660   gvn.transform(phi2);
 661   gvn.transform(region);
 662   gvn.transform(phi);
 663 
 664   kit->set_control(region);
 665   kit->set_memory(in_mem, alias_idx);
 666   kit->insert_mem_bar(Op_MemBarCPUOrder);
 667 
 668   return phi;
 669 }
 670 
 671 Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const {
 672   PhaseGVN& gvn = kit->gvn();
 673   Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed);
 674   Node* transformed_barrier = gvn.transform(barrier);
 675 
 676   if (transformed_barrier->is_LoadBarrier()) {
 677     if (barrier == transformed_barrier) {
 678       kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)));
 679     }
 680     Node* result = gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop));
 681     return result;
 682   } else {
 683     return val;
 684   }
 685 }
 686 
 687 static bool barrier_needed(C2Access& access) {
 688   return ZBarrierSet::barrier_needed(access.decorators(), access.type());
 689 }
 690 
 691 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
 692   Node* p = BarrierSetC2::load_at_resolved(access, val_type);
 693   if (!barrier_needed(access)) {
 694     return p;
 695   }
 696 
 697   bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
 698 
 699   assert(access.is_parse_access(), "entry not supported at optimization time");
 700   C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
 701   GraphKit* kit = parse_access.kit();
 702   PhaseGVN& gvn = kit->gvn();
 703   Node* adr = access.addr().node();
 704   Node* heap_base_oop = access.base();
 705   bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0;
 706   if (unsafe) {
 707     if (!ZVerifyLoadBarriers) {
 708       p = load_barrier(kit, p, adr);
 709     } else {
 710       if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) {
 711         p = load_barrier(kit, p, adr);
 712       } else {
 713         IdealKit ideal(kit);
 714         IdealVariable res(ideal);
 715 #define __ ideal.
 716         __ declarations_done();
 717         __ set(res, p);
 718         __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); {
 719           kit->sync_kit(ideal);
 720           p = load_barrier(kit, p, adr);
 721           __ set(res, p);
 722           __ sync_kit(kit);
 723         } __ end_if();
 724         kit->final_sync(ideal);
 725         p = __ value(res);
 726 #undef __
 727       }
 728     }
 729     return p;
 730   } else {
 731     return load_barrier(parse_access.kit(), p, access.addr().node(), weak, true, true);
 732   }

 733 }
 734 
 735 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 736                                                     Node* new_val, const Type* val_type) const {
 737   Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
 738   if (!barrier_needed(access)) {
 739     return result;

 740   }
 741 
 742   access.set_needs_pinning(false);
 743   return make_cmpx_loadbarrier(access);
 744 }
 745 
 746 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 747                                                      Node* new_val, const Type* value_type) const {
 748   Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
 749   if (!barrier_needed(access)) {
 750     return result;
 751   }
 752 
 753   Node* load_store = access.raw_access();
 754   bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0;
 755   bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR);
 756 
 757   if (!expected_is_null) {
 758     if (weak_cas) {
 759       access.set_needs_pinning(false);
 760       load_store = make_cas_loadbarrier(access);
 761     } else {
 762       access.set_needs_pinning(false);
 763       load_store = make_cas_loadbarrier(access);
 764     }
 765   }
 766 
 767   return load_store;
 768 }
 769 
 770 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
 771   Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
 772   if (!barrier_needed(access)) {
 773     return result;

 774   }
 775 
 776   Node* load_store = access.raw_access();
 777   Node* adr = access.addr().node();
 778 
 779   assert(access.is_parse_access(), "entry not supported at optimization time");
 780   C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
 781   return load_barrier(parse_access.kit(), load_store, adr, false, false, false);
 782 }
 783 
 784 // == Macro Expansion ==
 785 

 786 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {
 787   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
 788   Node* in_mem  = barrier->in(LoadBarrierNode::Memory);
 789   Node* in_val  = barrier->in(LoadBarrierNode::Oop);
 790   Node* in_adr  = barrier->in(LoadBarrierNode::Address);
 791 
 792   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
 793   Node* out_res  = barrier->proj_out(LoadBarrierNode::Oop);
 794 
 795   PhaseIterGVN &igvn = phase->igvn();
 796 
 797   if (ZVerifyLoadBarriers) {
 798     igvn.replace_node(out_res, in_val);
 799     igvn.replace_node(out_ctrl, in_ctrl);
 800     return;
 801   }
 802 
 803   if (barrier->can_be_eliminated()) {
 804     // Clone and pin the load for this barrier below the dominating
 805     // barrier: the load cannot be allowed to float above the
 806     // dominating barrier
 807     Node* load = in_val;
 808 
 809     if (load->is_Load()) {
 810       Node* new_load = load->clone();
 811       Node* addp = new_load->in(MemNode::Address);
 812       assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address");
 813       Node* cast = new CastPPNode(addp, igvn.type(addp), true);
 814       Node* ctrl = NULL;
 815       Node* similar = barrier->in(LoadBarrierNode::Similar);
 816       if (similar->is_Phi()) {
 817         // already expanded
 818         ctrl = similar->in(0);
 819       } else {
 820         assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape");
 821         ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control);
 822       }
 823       assert(ctrl != NULL, "bad control");
 824       cast->set_req(0, ctrl);
 825       igvn.transform(cast);
 826       new_load->set_req(MemNode::Address, cast);
 827       igvn.transform(new_load);
 828 
 829       igvn.replace_node(out_res, new_load);
 830       igvn.replace_node(out_ctrl, in_ctrl);
 831       return;
 832     }
 833     // cannot eliminate
 834   }
 835 
 836   // There are two cases that require the basic loadbarrier
 837   // 1) When the writeback of a healed oop must be avoided (swap)
 838   // 2) When we must guarantee that no reload of is done (swap, cas, cmpx)
 839   if (!barrier->is_writeback()) {
 840     assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop");
 841   }
 842 
 843   if (!barrier->oop_reload_allowed()) {
 844     expand_loadbarrier_basic(phase, barrier);
 845   } else {
 846     expand_loadbarrier_optimized(phase, barrier);
 847   }
 848 }
 849 
 850 // Basic loadbarrier using conventional argument passing
 851 void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
 852   PhaseIterGVN &igvn = phase->igvn();
 853 
 854   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
 855   Node* in_mem  = barrier->in(LoadBarrierNode::Memory);
 856   Node* in_val  = barrier->in(LoadBarrierNode::Oop);
 857   Node* in_adr  = barrier->in(LoadBarrierNode::Address);
 858 
 859   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
 860   Node* out_res  = barrier->proj_out(LoadBarrierNode::Oop);
 861 
 862   float unlikely  = PROB_UNLIKELY(0.999);
 863   const Type* in_val_maybe_null_t = igvn.type(in_val);
 864 
 865   Node* jthread = igvn.transform(new ThreadLocalNode());
 866   Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
 867   Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
 868   Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
 869   Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
 870   Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
 871   Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
 872   IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
 873   Node* then = igvn.transform(new IfTrueNode(iff));
 874   Node* elsen = igvn.transform(new IfFalseNode(iff));
 875 
 876   Node* result_region;
 877   Node* result_val;
 878 
 879   result_region = new RegionNode(3);
 880   result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);
 881 
 882   result_region->set_req(1, elsen);
 883   Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));
 884   res->init_req(0, elsen);
 885   result_val->set_req(1, res);
 886 
 887   const TypeFunc *tf = load_barrier_Type();
 888   Node* call;
 889   if (barrier->is_weak()) {
 890     call = new CallLeafNode(tf,
 891                             ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(),
 892                             "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded",
 893                             TypeRawPtr::BOTTOM);
 894   } else {
 895     call = new CallLeafNode(tf,
 896                             ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(),
 897                             "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded",
 898                             TypeRawPtr::BOTTOM);
 899   }
 900 
 901   call->init_req(TypeFunc::Control, then);
 902   call->init_req(TypeFunc::I_O    , phase->top());
 903   call->init_req(TypeFunc::Memory , in_mem);
 904   call->init_req(TypeFunc::FramePtr, phase->top());
 905   call->init_req(TypeFunc::ReturnAdr, phase->top());
 906   call->init_req(TypeFunc::Parms+0, in_val);
 907   if (barrier->is_writeback()) {
 908     call->init_req(TypeFunc::Parms+1, in_adr);
 909   } else {
 910     // When slow path is called with a null address, the healed oop will not be written back
 911     call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT));
 912   }
 913   call = igvn.transform(call);
 914 
 915   Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control));
 916   res = igvn.transform(new ProjNode(call, TypeFunc::Parms));
 917   res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t));
 918 
 919   result_region->set_req(2, ctrl);
 920   result_val->set_req(2, res);
 921 
 922   result_region = igvn.transform(result_region);
 923   result_val = igvn.transform(result_val);
 924 
 925   if (out_ctrl != NULL) { // Added if cond
 926     igvn.replace_node(out_ctrl, result_region);
 927   }
 928   igvn.replace_node(out_res, result_val);
 929 }
 930 
 931 // Optimized, low spill, loadbarrier variant using stub specialized on register used
 932 void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
 933   PhaseIterGVN &igvn = phase->igvn();
 934 #ifdef PRINT_NODE_TRAVERSALS
 935   Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop);
 936 #endif
 937 
 938   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
 939   Node* in_mem = barrier->in(LoadBarrierNode::Memory);
 940   Node* in_val = barrier->in(LoadBarrierNode::Oop);
 941   Node* in_adr = barrier->in(LoadBarrierNode::Address);
 942 
 943   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
 944   Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
 945 
 946   assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
 947 
 948 #ifdef PRINT_NODE_TRAVERSALS
 949   tty->print("\n\n\nBefore barrier optimization:\n");
 950   traverse(barrier, out_ctrl, out_res, -1);
 951 
 952   tty->print("\nBefore barrier optimization:  preceding_barrier_node\n");
 953   traverse(preceding_barrier_node, out_ctrl, out_res, -1);
 954 #endif
 955 
 956   float unlikely  = PROB_UNLIKELY(0.999);
 957 
 958   Node* jthread = igvn.transform(new ThreadLocalNode());
 959   Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
 960   Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
 961                                                  TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
 962                                                  MemNode::unordered));
 963   Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
 964   Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
 965   Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
 966   Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
 967   IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
 968   Node* then = igvn.transform(new IfTrueNode(iff));
 969   Node* elsen = igvn.transform(new IfFalseNode(iff));
 970 
 971   Node* slow_path_surrogate;
 972   if (!barrier->is_weak()) {
 973     slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
 974                                                                     (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
 975   } else {
 976     slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
 977                                                                         (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
 978   }
 979 
 980   Node *new_loadp;
 981   new_loadp = slow_path_surrogate;
 982   // Create the final region/phi pair to converge cntl/data paths to downstream code
 983   Node* result_region = igvn.transform(new RegionNode(3));
 984   result_region->set_req(1, then);
 985   result_region->set_req(2, elsen);
 986 
 987   Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
 988   result_phi->set_req(1, new_loadp);
 989   result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
 990 
 991   // Finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution
 992   // igvn.replace_node(out_ctrl, result_region);
 993   if (out_ctrl != NULL) { // added if cond
 994     igvn.replace_node(out_ctrl, result_region);
 995   }
 996   igvn.replace_node(out_res, result_phi);
 997 
 998   assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
 999 
1000 #ifdef PRINT_NODE_TRAVERSALS
1001   tty->print("\nAfter barrier optimization:  old out_ctrl\n");
1002   traverse(out_ctrl, out_ctrl, out_res, -1);
1003   tty->print("\nAfter barrier optimization:  old out_res\n");
1004   traverse(out_res, out_ctrl, out_res, -1);
1005   tty->print("\nAfter barrier optimization:  old barrier\n");
1006   traverse(barrier, out_ctrl, out_res, -1);
1007   tty->print("\nAfter barrier optimization:  preceding_barrier_node\n");
1008   traverse(preceding_barrier_node, result_region, result_phi, -1);
1009 #endif
1010 
1011   assert(is_gc_barrier_node(result_phi), "sanity");
1012   assert(step_over_gc_barrier(result_phi) == in_val, "sanity");


1013 }
1014 
1015 bool ZBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
1016   ZBarrierSetC2State* s = state();
1017   if (s->load_barrier_count() > 0) {
1018     PhaseMacroExpand macro(igvn);
1019 #ifdef ASSERT
1020     verify_gc_barriers(false);
1021 #endif
1022     int skipped = 0;
1023     while (s->load_barrier_count() > skipped) {
1024       int load_barrier_count = s->load_barrier_count();
1025       LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
1026       if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
1027         // Node is unreachable, so don't try to expand it
1028         s->remove_load_barrier_node(n);
1029         continue;
1030       }
1031       if (!n->can_be_eliminated()) {
1032         skipped++;
1033         continue;
1034       }
1035       expand_loadbarrier_node(&macro, n);
1036       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1037       if (C->failing()) {
1038         return true;
1039       }
1040     }
1041     while (s->load_barrier_count() > 0) {
1042       int load_barrier_count = s->load_barrier_count();
1043       LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);
1044       assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");
1045       assert(!n->can_be_eliminated(), "should have been processed already");
1046       expand_loadbarrier_node(&macro, n);
1047       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1048       if (C->failing()) {
1049         return true;
1050       }
1051     }
1052     igvn.set_delay_transform(false);
1053     igvn.optimize();
1054     if (C->failing()) {
1055       return true;
1056     }
1057   }
1058 
1059   return false;
1060 }
1061 
1062 // == Loop optimization ==
1063 
1064 static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1065   PhaseIterGVN &igvn = phase->igvn();
1066   Compile* C = Compile::current();
1067 
1068   LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round);
1069   if (lb2 == NULL) {
1070     return false;
1071   }
1072 
1073   if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) {
1074     assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), "Invalid address");
1075     igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop));
1076     C->set_major_progress();
1077     return false;
1078   }
1079 
1080   // That transformation may cause the Similar edge on dominated load barriers to be invalid
1081   lb->fix_similar_in_uses(&igvn);
1082 
1083   Node* val = lb->proj_out(LoadBarrierNode::Oop);
1084   assert(lb2->has_true_uses(), "Invalid uses");
1085   assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), "Invalid oop");
1086   phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1087   phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1088   igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop));
1089 
1090   return true;
1091 }
1092 
1093 static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) {
1094   assert(dom->is_Region() || i == -1, "");
1095 
1096   Node* m = mem;
1097   while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) {
1098     if (m->is_Mem()) {
1099       assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, "");
1100       m = m->in(MemNode::Memory);
1101     } else if (m->is_MergeMem()) {
1102       m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1103     } else if (m->is_Phi()) {
1104       if (m->in(0) == dom && i != -1) {
1105         m = m->in(i);
1106         break;
1107       } else {
1108         m = m->in(LoopNode::EntryControl);
1109       }
1110     } else if (m->is_Proj()) {
1111       m = m->in(0);
1112     } else if (m->is_SafePoint() || m->is_MemBar()) {
1113       m = m->in(TypeFunc::Memory);
1114     } else {
1115 #ifdef ASSERT
1116       m->dump();
1117 #endif
1118       ShouldNotReachHere();
1119     }
1120   }
1121 
1122   return m;
1123 }
1124 
1125 static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) {
1126   PhaseIterGVN &igvn = phase->igvn();
1127   Compile* C = Compile::current();
1128   Node* the_clone = lb->clone();
1129   the_clone->set_req(LoadBarrierNode::Control, ctl);
1130   the_clone->set_req(LoadBarrierNode::Memory, mem);
1131   if (oop_in != NULL) {
1132     the_clone->set_req(LoadBarrierNode::Oop, oop_in);
1133   }
1134 
1135   LoadBarrierNode* new_lb = the_clone->as_LoadBarrier();
1136   igvn.register_new_node_with_optimizer(new_lb);
1137   IdealLoopTree *loop = phase->get_loop(new_lb->in(0));
1138   phase->set_ctrl(new_lb, new_lb->in(0));
1139   phase->set_loop(new_lb, loop);
1140   phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1);
1141   if (!loop->_child) {
1142     loop->_body.push(new_lb);
1143   }
1144 
1145   Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control);
1146   igvn.register_new_node_with_optimizer(proj_ctl);
1147   phase->set_ctrl(proj_ctl, proj_ctl->in(0));
1148   phase->set_loop(proj_ctl, loop);
1149   phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1);
1150   if (!loop->_child) {
1151     loop->_body.push(proj_ctl);
1152   }
1153 
1154   Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop);
1155   phase->register_new_node(proj_oop, new_lb);
1156 
1157   if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) {
1158     LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier();
1159     if (!phase->is_dominator(similar, ctl)) {
1160       igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top());
1161     }
1162   }
1163 
1164   return new_lb;
1165 }
1166 
1167 static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) {
1168   PhaseIterGVN &igvn = phase->igvn();
1169   Node* val = lb->proj_out(LoadBarrierNode::Oop);
1170   igvn.replace_node(val, new_val);
1171   phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1172   phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1173 }
1174 
1175 static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1176   PhaseIterGVN &igvn = phase->igvn();
1177   Compile* C = Compile::current();
1178 
1179   if (lb->in(LoadBarrierNode::Oop)->is_Phi()) {
1180     Node* oop_phi = lb->in(LoadBarrierNode::Oop);
1181 
1182     if ((oop_phi->req() != 3) || (oop_phi->in(2) == oop_phi)) {
1183       // Ignore phis with only one input
1184       return false;
1185     }
1186 
1187     if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)),
1188                             oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) {
1189       // That transformation may cause the Similar edge on dominated load barriers to be invalid
1190       lb->fix_similar_in_uses(&igvn);
1191 
1192       RegionNode* region = oop_phi->in(0)->as_Region();
1193 
1194       int backedge = LoopNode::LoopBackControl;
1195       if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) {
1196         Node* c = region->in(backedge)->in(0)->in(0);
1197         assert(c->unique_ctrl_out() == region->in(backedge)->in(0), "");
1198         Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge);
1199         Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop;
1200         if (!phase->is_dominator(oop_c, c)) {
1201           return false;
1202         }
1203       }
1204 
1205       // If the node on the backedge above the phi is the node itself - we have a self loop.
1206       // Don't clone - this will be folded later.
1207       if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) {
1208         return false;
1209       }
1210 
1211       bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined();
1212       Node *phi = oop_phi->clone();
1213 
1214       for (uint i = 1; i < region->req(); i++) {
1215         Node* ctrl = region->in(i);
1216         if (ctrl != C->top()) {
1217           assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), "");
1218 
1219           Node* mem = lb->in(LoadBarrierNode::Memory);
1220           Node* m = find_dominating_memory(phase, mem, region, i);
1221 
1222           if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) {
1223             ctrl = ctrl->in(0)->in(0);
1224           } else if (region->is_Loop() && is_strip_mined) {
1225             // If this is a strip mined loop, control must move above OuterStripMinedLoop
1226             assert(i == LoopNode::EntryControl, "check");
1227             assert(ctrl->is_OuterStripMinedLoop(), "sanity");
1228             ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl);
1229           }
1230 
1231           LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i));
1232           Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control);
1233 
1234           if (is_strip_mined && (i == LoopNode::EntryControl)) {
1235             assert(region->in(i)->is_OuterStripMinedLoop(), "");
1236             igvn.replace_input_of(region->in(i), i, out_ctrl);
1237             phase->set_idom(region->in(i), out_ctrl, phase->dom_depth(out_ctrl));
1238           } else if (ctrl == region->in(i)) {
1239             igvn.replace_input_of(region, i, out_ctrl);
1240             // Only update the idom if is the loop entry we are updating
1241             // - A loop backedge doesn't change the idom
1242             if (region->is_Loop() && i == LoopNode::EntryControl) {
1243               phase->set_idom(region, out_ctrl, phase->dom_depth(out_ctrl));
1244             }
1245           } else {
1246             Node* iff = region->in(i)->in(0);
1247             igvn.replace_input_of(iff, 0, out_ctrl);
1248             phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1);
1249           }
1250           phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop));
1251         }
1252       }
1253       phase->register_new_node(phi, region);
1254       replace_barrier(phase, lb, phi);
1255 
1256       if (region->is_Loop()) {
1257         // Load barrier moved to the back edge of the Loop may now
1258         // have a safepoint on the path to the barrier on the Similar
1259         // edge
1260         igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top());
1261         Node* head = region->in(LoopNode::EntryControl);
1262         phase->set_idom(region, head, phase->dom_depth(head)+1);
1263         phase->recompute_dom_depth();
1264         if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) {
1265           head->as_CountedLoop()->set_normal_loop();
1266         }
1267       }
1268 
1269       return true;
1270     }
1271   }
1272 
1273   return false;
1274 }
1275 
1276 static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1277   PhaseIterGVN &igvn = phase->igvn();
1278   IdealLoopTree *lb_loop = phase->get_loop(lb->in(0));
1279   if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) {
1280     Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop));
1281     IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl);
1282     IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address)));
1283     if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) {
1284       // That transformation may cause the Similar edge on dominated load barriers to be invalid
1285       lb->fix_similar_in_uses(&igvn);
1286 
1287       Node* head = lb_loop->_head;
1288       assert(head->is_Loop(), "");
1289 
1290       if (phase->is_dominator(head, oop_ctrl)) {
1291         assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, "");
1292         assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), "");
1293         return false;
1294       }
1295 
1296       if (head->is_CountedLoop()) {
1297         CountedLoopNode* cloop = head->as_CountedLoop();
1298         if (cloop->is_main_loop()) {
1299           cloop->set_normal_loop();
1300         }
1301         // When we are moving barrier out of a counted loop,
1302         // make sure we move it all the way out of the strip mined outer loop.
1303         if (cloop->is_strip_mined()) {
1304           head = cloop->outer_loop();
1305         }
1306       }
1307 
1308       Node* mem = lb->in(LoadBarrierNode::Memory);
1309       Node* m = find_dominating_memory(phase, mem, head, -1);
1310 
1311       LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL);
1312 
1313       assert(phase->idom(head) == head->in(LoopNode::EntryControl), "");
1314       Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1315       igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl);
1316       phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1);
1317 
1318       replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop));
1319 
1320       phase->recompute_dom_depth();
1321 
1322       return true;
1323     }
1324   }
1325 
1326   return false;
1327 }
1328 
1329 static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1330   PhaseIterGVN &igvn = phase->igvn();
1331   Node* in_val = lb->in(LoadBarrierNode::Oop);
1332   for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) {
1333     Node* u = in_val->fast_out(i);
1334     if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
1335       Node* this_ctrl = lb->in(LoadBarrierNode::Control);
1336       Node* other_ctrl = u->in(LoadBarrierNode::Control);
1337 
1338       Node* lca = phase->dom_lca(this_ctrl, other_ctrl);
1339       bool ok = true;
1340 
1341       Node* proj1 = NULL;
1342       Node* proj2 = NULL;
1343 
1344       while (this_ctrl != lca && ok) {
1345         if (this_ctrl->in(0) != NULL &&
1346             this_ctrl->in(0)->is_MultiBranch()) {
1347           if (this_ctrl->in(0)->in(0) == lca) {
1348             assert(proj1 == NULL, "");
1349             assert(this_ctrl->is_Proj(), "");
1350             proj1 = this_ctrl;
1351           } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1352             ok = false;
1353           }
1354         }
1355         this_ctrl = phase->idom(this_ctrl);
1356       }
1357       while (other_ctrl != lca && ok) {
1358         if (other_ctrl->in(0) != NULL &&
1359             other_ctrl->in(0)->is_MultiBranch()) {
1360           if (other_ctrl->in(0)->in(0) == lca) {
1361             assert(other_ctrl->is_Proj(), "");
1362             assert(proj2 == NULL, "");
1363             proj2 = other_ctrl;
1364           } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1365             ok = false;
1366           }
1367         }
1368         other_ctrl = phase->idom(other_ctrl);
1369       }
1370       assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), "");
1371       if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) {
1372         // That transformation may cause the Similar edge on dominated load barriers to be invalid
1373         lb->fix_similar_in_uses(&igvn);
1374         u->as_LoadBarrier()->fix_similar_in_uses(&igvn);
1375 
1376         Node* split = lca->unique_ctrl_out();
1377         assert(split->in(0) == lca, "");
1378 
1379         Node* mem = lb->in(LoadBarrierNode::Memory);
1380         Node* m = find_dominating_memory(phase, mem, split, -1);
1381         LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL);
1382 
1383         Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1384         igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control));
1385         phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1);
1386 
1387         Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop);
1388         replace_barrier(phase, lb, proj_oop);
1389         replace_barrier(phase, u->as_LoadBarrier(), proj_oop);
1390 
1391         phase->recompute_dom_depth();
1392 
1393         return true;
1394       }
1395     }
1396   }
1397 
1398   return false;
1399 }
1400 
1401 void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) {
1402   if (!Compile::current()->directive()->ZOptimizeLoadBarriersOption) {
1403     return;
1404   }
1405 
1406   if (!node->is_LoadBarrier()) {
1407     return;
1408   }
1409 
1410   if (!node->as_LoadBarrier()->has_true_uses()) {
1411     return;
1412   }
1413 
1414   if (replace_with_dominating_barrier(phase, node->as_LoadBarrier(), last_round)) {
1415     return;
1416   }
1417 
1418   if (split_barrier_thru_phi(phase, node->as_LoadBarrier())) {
1419     return;
1420   }
1421 
1422   if (move_out_of_loop(phase, node->as_LoadBarrier())) {
1423     return;
1424   }
1425 
1426   if (common_barriers(phase, node->as_LoadBarrier())) {
1427     return;
1428   }
1429 }
1430 
1431 Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const {
1432   Node* node = c;
1433 
1434   // 1. This step follows potential oop projections of a load barrier before expansion
1435   if (node->is_Proj()) {
1436     node = node->in(0);
1437   }
1438 
1439   // 2. This step checks for unexpanded load barriers
1440   if (node->is_LoadBarrier()) {
1441     return node->in(LoadBarrierNode::Oop);
1442   }
1443 
1444   // 3. This step checks for the phi corresponding to an optimized load barrier expansion
1445   if (node->is_Phi()) {
1446     PhiNode* phi = node->as_Phi();
1447     Node* n = phi->in(1);
1448     if (n != NULL && (n->is_LoadBarrierSlowReg() ||  n->is_LoadBarrierWeakSlowReg())) {
1449       assert(c == node, "projections from step 1 should only be seen before macro expansion");
1450       return phi->in(2);
1451     }
1452   }
1453 
1454   return c;
1455 }
1456 
1457 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
1458   return type == T_OBJECT || type == T_ARRAY;
1459 }
1460 
1461 bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
1462   if (opcode != Op_LoadBarrierSlowReg &&
1463       opcode != Op_LoadBarrierWeakSlowReg) {
1464     return false;
1465   }
1466 



1467 #ifdef ASSERT
1468   if (VerifyOptoOopOffsets) {
1469     MemNode* mem  = n->as_Mem();
1470     // Check to see if address types have grounded out somehow.
1471     const TypeInstPtr* tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
1472     ciInstanceKlass* k = tp->klass()->as_instance_klass();
1473     bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
1474     assert(!tp || oop_offset_is_sane, "");





1475   }
1476 #endif
1477 
1478   return true;
1479 }
1480 
1481 bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
1482   if (opcode == Op_CallLeaf &&
1483       (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() ||
1484        n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr())) {
1485     mem_op = true;
1486     mem_addr_idx = TypeFunc::Parms + 1;
1487     return true;





1488   }

1489 
1490   return false;






























1491 }
1492 
1493 // == Verification ==
1494 
1495 #ifdef ASSERT
1496 
1497 static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
1498   if (visited.test_set(n->_idx)) {
1499     return true;
1500   }
1501 
1502   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1503     Node* u = n->fast_out(i);
1504     if (u->is_LoadBarrier()) {
1505     } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) {
1506       if (!look_for_barrier(u, post_parse, visited)) {
1507         return false;
1508       }
1509     } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) {
1510       if (!look_for_barrier(u, post_parse, visited)) {
1511         return false;
1512       }
1513     } else if (u->Opcode() != Op_SCMemProj) {
1514       tty->print("bad use"); u->dump();
1515       return false;
1516     }
1517   }
1518 
1519   return true;
1520 }
1521 
1522 void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
1523   if (phase == BarrierSetC2::BeforeCodeGen) return;
1524   bool post_parse = phase == BarrierSetC2::BeforeOptimize;
1525   verify_gc_barriers(post_parse);













1526 }
1527 


1528 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
1529   ZBarrierSetC2State* s = state();
1530   Compile* C = Compile::current();
1531   ResourceMark rm;
1532   VectorSet visited(Thread::current()->resource_area());

1533   for (int i = 0; i < s->load_barrier_count(); i++) {
1534     LoadBarrierNode* n = s->load_barrier_node(i);
1535 
1536     // The dominating barrier on the same address if it exists and
1537     // this barrier must not be applied on the value from the same
1538     // load otherwise the value is not reloaded before it's used the
1539     // second time.
1540     assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1541            (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1542             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
1543             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
1544            "broken similar edge");
1545 
1546     assert(post_parse || n->as_LoadBarrier()->has_true_uses(),
1547            "found unneeded load barrier");
1548 
1549     // Several load barrier nodes chained through their Similar edge
1550     // break the code that remove the barriers in final graph reshape.
1551     assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1552            (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1553             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),
1554            "chain of Similar load barriers");
1555 
1556     if (!n->in(LoadBarrierNode::Similar)->is_top()) {
1557       ResourceMark rm;
1558       Unique_Node_List wq;
1559       Node* other = n->in(LoadBarrierNode::Similar)->in(0);
1560       wq.push(n);
1561       bool ok = true;
1562       bool dom_found = false;
1563       for (uint next = 0; next < wq.size(); ++next) {
1564         Node *n = wq.at(next);
1565         assert(n->is_CFG(), "");
1566         assert(!n->is_SafePoint(), "");
1567 
1568         if (n == other) {
1569           continue;
1570         }
1571 
1572         if (n->is_Region()) {
1573           for (uint i = 1; i < n->req(); i++) {
1574             Node* m = n->in(i);
1575             if (m != NULL) {
1576               wq.push(m);
1577             }
1578           }
1579         } else {
1580           Node* m = n->in(0);
1581           if (m != NULL) {
1582             wq.push(m);
1583           }
1584         }
1585       }
1586     }









































































































1587 
1588     if (ZVerifyLoadBarriers) {
1589       if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) {
1590         visited.Clear();
1591         bool found = look_for_barrier(n, post_parse, visited);
1592         if (!found) {
1593           n->dump(1);
1594           n->dump(-3);
1595           stringStream ss;
1596           C->method()->print_short_name(&ss);
1597           tty->print_cr("-%s-", ss.as_string());
1598           assert(found, "");












































































1599         }
1600       }






























































































































1601     }

















1602   }














1603 }
1604 
1605 #endif


































































































































































































































































1606 
1607 bool ZBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
1608   switch (opcode) {
1609     case Op_LoadBarrierSlowReg:
1610     case Op_LoadBarrierWeakSlowReg:
1611       conn_graph->add_objload_to_connection_graph(n, delayed_worklist);
1612       return true;
1613 
1614     case Op_Proj:
1615       if (n->as_Proj()->_con != LoadBarrierNode::Oop || !n->in(0)->is_LoadBarrier()) {
1616         return false;



1617       }
1618       conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), delayed_worklist);
1619       return true;
1620   }
1621 
1622   return false;
























1623 }
1624 
1625 bool ZBarrierSetC2::escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const {
1626   switch (opcode) {
1627     case Op_LoadBarrierSlowReg:
1628     case Op_LoadBarrierWeakSlowReg:
1629       if (gvn->type(n)->make_ptr() == NULL) {
1630         return false;































1631       }
1632       conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(MemNode::Address), NULL);
1633       return true;




































































1634 
1635     case Op_Proj:
1636       if (n->as_Proj()->_con != LoadBarrierNode::Oop || !n->in(0)->is_LoadBarrier()) {
1637         return false;



1638       }
1639       conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), NULL);
1640       return true;
1641   }
1642 
1643   return false;




























1644 }
< prev index next >