1 /*
   2  * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "opto/compile.hpp"
  26 #include "opto/castnode.hpp"
  27 #include "opto/graphKit.hpp"
  28 #include "opto/idealKit.hpp"
  29 #include "opto/loopnode.hpp"
  30 #include "opto/macro.hpp"
  31 #include "opto/node.hpp"
  32 #include "opto/type.hpp"
  33 #include "utilities/macros.hpp"
  34 #include "gc/z/zBarrierSet.hpp"
  35 #include "gc/z/c2/zBarrierSetC2.hpp"
  36 #include "gc/z/zThreadLocalData.hpp"
  37 #include "gc/z/zBarrierSetRuntime.hpp"
  38 
  39 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena)
  40   : _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8,  0, NULL)) {}
  41 
  42 int ZBarrierSetC2State::load_barrier_count() const {
  43   return _load_barrier_nodes->length();
  44 }
  45 
  46 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {
  47   assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");
  48   _load_barrier_nodes->append(n);
  49 }
  50 
  51 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {
  52   // this function may be called twice for a node so check
  53   // that the node is in the array before attempting to remove it
  54   if (_load_barrier_nodes->contains(n)) {
  55     _load_barrier_nodes->remove(n);
  56   }
  57 }
  58 
  59 LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const {
  60   return _load_barrier_nodes->at(idx);
  61 }
  62 
  63 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
  64   return new(comp_arena) ZBarrierSetC2State(comp_arena);
  65 }
  66 
  67 ZBarrierSetC2State* ZBarrierSetC2::state() const {
  68   return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
  69 }
  70 
  71 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {
  72   // 1. This step follows potential oop projections of a load barrier before expansion
  73   if (node->is_Proj()) {
  74     node = node->in(0);
  75   }
  76 
  77   // 2. This step checks for unexpanded load barriers
  78   if (node->is_LoadBarrier()) {
  79     return true;
  80   }
  81 
  82   // 3. This step checks for the phi corresponding to an optimized load barrier expansion
  83   if (node->is_Phi()) {
  84     PhiNode* phi = node->as_Phi();
  85     Node* n = phi->in(1);
  86     if (n != NULL && (n->is_LoadBarrierSlowReg() ||  n->is_LoadBarrierWeakSlowReg())) {
  87       return true;
  88     }
  89   }
  90 
  91   return false;
  92 }
  93 
  94 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {
  95   if (node->is_LoadBarrier()) {
  96     state()->add_load_barrier_node(node->as_LoadBarrier());
  97   }
  98 }
  99 
 100 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
 101   if (node->is_LoadBarrier()) {
 102     state()->remove_load_barrier_node(node->as_LoadBarrier());
 103   }
 104 }
 105 
 106 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const {
 107   // Remove useless LoadBarrier nodes
 108   ZBarrierSetC2State* s = state();
 109   for (int i = s->load_barrier_count()-1; i >= 0; i--) {
 110     LoadBarrierNode* n = s->load_barrier_node(i);
 111     if (!useful.member(n)) {
 112       unregister_potential_barrier_node(n);
 113     }
 114   }
 115 }
 116 
 117 void ZBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
 118   if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
 119     igvn->_worklist.push(node);
 120   }
 121 }
 122 
 123 void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) {
 124   // Look for dominating barriers on the same address only once all
 125   // other loop opts are over: loop opts may cause a safepoint to be
 126   // inserted between a barrier and its dominating barrier.
 127   Compile* C = Compile::current();
 128   ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2();
 129   ZBarrierSetC2State* s = bs->state();
 130   if (s->load_barrier_count() >= 2) {
 131     Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]);
 132     PhaseIdealLoop ideal_loop(igvn, LoopOptsLastRound);
 133     if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
 134   }
 135 }
 136 
 137 void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {
 138   // Permanent temporary workaround
 139   // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is
 140   // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to
 141   // the worklist. Unless we add them explicitly they are not guaranteed to end up there.
 142   ZBarrierSetC2State* s = state();
 143 
 144   for (int i = 0; i < s->load_barrier_count(); i++) {
 145     LoadBarrierNode* n = s->load_barrier_node(i);
 146     worklist->push(n);
 147   }
 148 }
 149 
 150 const TypeFunc* ZBarrierSetC2::load_barrier_Type() const {
 151   const Type** fields;
 152 
 153   // Create input types (domain)
 154   fields = TypeTuple::fields(2);
 155   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
 156   fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM;
 157   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
 158 
 159   // Create result type (range)
 160   fields = TypeTuple::fields(1);
 161   fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM;
 162   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
 163 
 164   return TypeFunc::make(domain, range);
 165 }
 166 
 167 // == LoadBarrierNode ==
 168 
 169 LoadBarrierNode::LoadBarrierNode(Compile* C,
 170                                  Node* c,
 171                                  Node* mem,
 172                                  Node* val,
 173                                  Node* adr,
 174                                  bool weak,
 175                                  bool writeback,
 176                                  bool oop_reload_allowed) :
 177     MultiNode(Number_of_Inputs),
 178     _weak(weak),
 179     _writeback(writeback),
 180     _oop_reload_allowed(oop_reload_allowed) {
 181   init_req(Control, c);
 182   init_req(Memory, mem);
 183   init_req(Oop, val);
 184   init_req(Address, adr);
 185   init_req(Similar, C->top());
 186 
 187   init_class_id(Class_LoadBarrier);
 188   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 189   bs->register_potential_barrier_node(this);
 190 }
 191 
 192 uint LoadBarrierNode::size_of() const {
 193   return sizeof(*this);
 194 }
 195 
 196 uint LoadBarrierNode::cmp(const Node& n) const {
 197   ShouldNotReachHere();
 198   return 0;
 199 }
 200 
 201 const Type *LoadBarrierNode::bottom_type() const {
 202   const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
 203   Node* in_oop = in(Oop);
 204   floadbarrier[Control] = Type::CONTROL;
 205   floadbarrier[Memory] = Type::MEMORY;
 206   floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();
 207   return TypeTuple::make(Number_of_Outputs, floadbarrier);
 208 }
 209 
 210 const TypePtr* LoadBarrierNode::adr_type() const {
 211   ShouldNotReachHere();
 212   return NULL;
 213 }
 214 
 215 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
 216   const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
 217   const Type* val_t = phase->type(in(Oop));
 218   floadbarrier[Control] = Type::CONTROL;
 219   floadbarrier[Memory] = Type::MEMORY;
 220   floadbarrier[Oop] = val_t;
 221   return TypeTuple::make(Number_of_Outputs, floadbarrier);
 222 }
 223 
 224 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
 225   if (phase != NULL) {
 226     return phase->is_dominator(d, n);
 227   }
 228 
 229   for (int i = 0; i < 10 && n != NULL; i++) {
 230     n = IfNode::up_one_dom(n, linear_only);
 231     if (n == d) {
 232       return true;
 233     }
 234   }
 235 
 236   return false;
 237 }
 238 
 239 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {
 240   Node* val = in(LoadBarrierNode::Oop);
 241   if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
 242     LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
 243     assert(lb->in(Address) == in(Address), "");
 244     // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
 245     if (lb->in(Oop) == in(Oop)) {
 246       return lb;
 247     }
 248     // Follow chain of load barrier through Similar edges
 249     while (!lb->in(Similar)->is_top()) {
 250       lb = lb->in(Similar)->in(0)->as_LoadBarrier();
 251       assert(lb->in(Address) == in(Address), "");
 252     }
 253     if (lb != in(Similar)->in(0)) {
 254       return lb;
 255     }
 256   }
 257   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
 258     Node* u = val->fast_out(i);
 259     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {
 260       Node* this_ctrl = in(LoadBarrierNode::Control);
 261       Node* other_ctrl = u->in(LoadBarrierNode::Control);
 262       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
 263         return u->as_LoadBarrier();
 264       }
 265     }
 266   }
 267 
 268   if (ZVerifyLoadBarriers || can_be_eliminated()) {
 269     return NULL;
 270   }
 271 
 272   if (!look_for_similar) {
 273     return NULL;
 274   }
 275 
 276   Node* addr = in(LoadBarrierNode::Address);
 277   for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
 278     Node* u = addr->fast_out(i);
 279     if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
 280       Node* this_ctrl = in(LoadBarrierNode::Control);
 281       Node* other_ctrl = u->in(LoadBarrierNode::Control);
 282       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
 283         ResourceMark rm;
 284         Unique_Node_List wq;
 285         wq.push(in(LoadBarrierNode::Control));
 286         bool ok = true;
 287         bool dom_found = false;
 288         for (uint next = 0; next < wq.size(); ++next) {
 289           Node *n = wq.at(next);
 290           if (n->is_top()) {
 291             return NULL;
 292           }
 293           assert(n->is_CFG(), "");
 294           if (n->is_SafePoint()) {
 295             ok = false;
 296             break;
 297           }
 298           if (n == u) {
 299             dom_found = true;
 300             continue;
 301           }
 302           if (n->is_Region()) {
 303             for (uint i = 1; i < n->req(); i++) {
 304               Node* m = n->in(i);
 305               if (m != NULL) {
 306                 wq.push(m);
 307               }
 308             }
 309           } else {
 310             Node* m = n->in(0);
 311             if (m != NULL) {
 312               wq.push(m);
 313             }
 314           }
 315         }
 316         if (ok) {
 317           assert(dom_found, "");
 318           return u->as_LoadBarrier();;
 319         }
 320         break;
 321       }
 322     }
 323   }
 324 
 325   return NULL;
 326 }
 327 
 328 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
 329   // Change to that barrier may affect a dominated barrier so re-push those
 330   Node* val = in(LoadBarrierNode::Oop);
 331 
 332   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
 333     Node* u = val->fast_out(i);
 334     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
 335       Node* this_ctrl = in(Control);
 336       Node* other_ctrl = u->in(Control);
 337       if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
 338         igvn->_worklist.push(u);
 339       }
 340     }
 341 
 342     Node* addr = in(LoadBarrierNode::Address);
 343     for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
 344       Node* u = addr->fast_out(i);
 345       if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {
 346         Node* this_ctrl = in(Control);
 347         Node* other_ctrl = u->in(Control);
 348         if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
 349           igvn->_worklist.push(u);
 350         }
 351       }
 352     }
 353   }
 354 }
 355 
 356 Node *LoadBarrierNode::Identity(PhaseGVN *phase) {
 357   if (!phase->C->directive()->ZOptimizeLoadBarriersOption) {
 358     return this;
 359   }
 360 
 361   bool redundant_addr = false;
 362   LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
 363   if (dominating_barrier != NULL) {
 364     assert(dominating_barrier->in(Oop) == in(Oop), "");
 365     return dominating_barrier;
 366   }
 367 
 368   return this;
 369 }
 370 
 371 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
 372   if (remove_dead_region(phase, can_reshape)) {
 373     return this;
 374   }
 375 
 376   Node* val = in(Oop);
 377   Node* mem = in(Memory);
 378   Node* ctrl = in(Control);
 379   Node* adr = in(Address);
 380   assert(val->Opcode() != Op_LoadN, "");
 381 
 382   if (mem->is_MergeMem()) {
 383     Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
 384     set_req(Memory, new_mem);
 385     if (mem->outcnt() == 0 && can_reshape) {
 386       phase->is_IterGVN()->_worklist.push(mem);
 387     }
 388 
 389     return this;
 390   }
 391 
 392   bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption;
 393   LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL;
 394   if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
 395     assert(in(Address) == dominating_barrier->in(Address), "");
 396     set_req(Similar, dominating_barrier->proj_out(Oop));
 397     return this;
 398   }
 399 
 400   bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) ||
 401                    (can_reshape && (dominating_barrier != NULL || !has_true_uses()));
 402 
 403   if (eliminate) {
 404     if (can_reshape) {
 405       PhaseIterGVN* igvn = phase->is_IterGVN();
 406       Node* out_ctrl = proj_out_or_null(Control);
 407       Node* out_res = proj_out_or_null(Oop);
 408 
 409       if (out_ctrl != NULL) {
 410         igvn->replace_node(out_ctrl, ctrl);
 411       }
 412 
 413       // That transformation may cause the Similar edge on the load barrier to be invalid
 414       fix_similar_in_uses(igvn);
 415       if (out_res != NULL) {
 416         if (dominating_barrier != NULL) {
 417           igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
 418         } else {
 419           igvn->replace_node(out_res, val);
 420         }
 421       }
 422     }
 423 
 424     return new ConINode(TypeInt::ZERO);
 425   }
 426 
 427   // If the Similar edge is no longer a load barrier, clear it
 428   Node* similar = in(Similar);
 429   if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
 430     set_req(Similar, phase->C->top());
 431     return this;
 432   }
 433 
 434   if (can_reshape) {
 435     // If this barrier is linked through the Similar edge by a
 436     // dominated barrier and both barriers have the same Oop field,
 437     // the dominated barrier can go away, so push it for reprocessing.
 438     // We also want to avoid a barrier to depend on another dominating
 439     // barrier through its Similar edge that itself depend on another
 440     // barrier through its Similar edge and rather have the first
 441     // depend on the third.
 442     PhaseIterGVN* igvn = phase->is_IterGVN();
 443     Node* out_res = proj_out(Oop);
 444     for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 445       Node* u = out_res->fast_out(i);
 446       if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
 447           (u->in(Oop) == val || !u->in(Similar)->is_top())) {
 448         igvn->_worklist.push(u);
 449       }
 450     }
 451 
 452     push_dominated_barriers(igvn);
 453   }
 454 
 455   return NULL;
 456 }
 457 
 458 uint LoadBarrierNode::match_edge(uint idx) const {
 459   ShouldNotReachHere();
 460   return 0;
 461 }
 462 
 463 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {
 464   Node* out_res = proj_out_or_null(Oop);
 465   if (out_res == NULL) {
 466     return;
 467   }
 468 
 469   for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 470     Node* u = out_res->fast_out(i);
 471     if (u->is_LoadBarrier() && u->in(Similar) == out_res) {
 472       igvn->replace_input_of(u, Similar, igvn->C->top());
 473       --i;
 474       --imax;
 475     }
 476   }
 477 }
 478 
 479 bool LoadBarrierNode::has_true_uses() const {
 480   Node* out_res = proj_out_or_null(Oop);
 481   if (out_res == NULL) {
 482     return false;
 483   }
 484 
 485   for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 486     Node* u = out_res->fast_out(i);
 487     if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
 488       return true;
 489     }
 490   }
 491 
 492   return false;
 493 }
 494 
 495 // == Accesses ==
 496 
 497 Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicAccess& access) const {
 498   assert(!UseCompressedOops, "Not allowed");
 499   CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access();
 500   PhaseGVN& gvn = access.kit()->gvn();
 501   Compile* C = Compile::current();
 502   GraphKit* kit = access.kit();
 503 
 504   Node* in_ctrl     = cas->in(MemNode::Control);
 505   Node* in_mem      = cas->in(MemNode::Memory);
 506   Node* in_adr      = cas->in(MemNode::Address);
 507   Node* in_val      = cas->in(MemNode::ValueIn);
 508   Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn);
 509 
 510   float likely                   = PROB_LIKELY(0.999);
 511 
 512   const TypePtr *adr_type        = gvn.type(in_adr)->isa_ptr();
 513   Compile::AliasType* alias_type = C->alias_type(adr_type);
 514   int alias_idx                  = C->get_alias_index(adr_type);
 515 
 516   // Outer check - true: continue, false: load and check
 517   Node* region   = new RegionNode(3);
 518   Node* phi      = new PhiNode(region, TypeInt::BOOL);
 519   Node* phi_mem  = new PhiNode(region, Type::MEMORY, adr_type);
 520 
 521   // Inner check - is the healed ref equal to the expected
 522   Node* region2  = new RegionNode(3);
 523   Node* phi2     = new PhiNode(region2, TypeInt::BOOL);
 524   Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type);
 525 
 526   // CAS node returns 0 or 1
 527   Node* cmp     = gvn.transform(new CmpINode(cas, kit->intcon(0)));
 528   Node* bol     = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
 529   IfNode* iff   = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
 530   Node* then    = gvn.transform(new IfTrueNode(iff));
 531   Node* elsen   = gvn.transform(new IfFalseNode(iff));
 532 
 533   Node* scmemproj1   = gvn.transform(new SCMemProjNode(cas));
 534 
 535   kit->set_memory(scmemproj1, alias_idx);
 536   phi_mem->init_req(1, scmemproj1);
 537   phi_mem2->init_req(2, scmemproj1);
 538 
 539   // CAS fail - reload and heal oop
 540   Node* reload      = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
 541   Node* barrier     = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
 542   Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
 543   Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
 544 
 545   // Check load
 546   Node* tmpX    = gvn.transform(new CastP2XNode(NULL, barrierdata));
 547   Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
 548   Node* cmp2    = gvn.transform(new CmpXNode(tmpX, in_expX));
 549   Node *bol2    = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
 550   IfNode* iff2  = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
 551   Node* then2   = gvn.transform(new IfTrueNode(iff2));
 552   Node* elsen2  = gvn.transform(new IfFalseNode(iff2));
 553 
 554   // redo CAS
 555   Node* cas2       = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order()));
 556   Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2));
 557   kit->set_control(elsen2);
 558   kit->set_memory(scmemproj2, alias_idx);
 559 
 560   // Merge inner flow - check if healed oop was equal too expected.
 561   region2->set_req(1, kit->control());
 562   region2->set_req(2, then2);
 563   phi2->set_req(1, cas2);
 564   phi2->set_req(2, kit->intcon(0));
 565   phi_mem2->init_req(1, scmemproj2);
 566   kit->set_memory(phi_mem2, alias_idx);
 567 
 568   // Merge outer flow - then check if first CAS succeeded
 569   region->set_req(1, then);
 570   region->set_req(2, region2);
 571   phi->set_req(1, kit->intcon(1));
 572   phi->set_req(2, phi2);
 573   phi_mem->init_req(2, phi_mem2);
 574   kit->set_memory(phi_mem, alias_idx);
 575 
 576   gvn.transform(region2);
 577   gvn.transform(phi2);
 578   gvn.transform(phi_mem2);
 579   gvn.transform(region);
 580   gvn.transform(phi);
 581   gvn.transform(phi_mem);
 582 
 583   kit->set_control(region);
 584   kit->insert_mem_bar(Op_MemBarCPUOrder);
 585 
 586   return phi;
 587 }
 588 
 589 Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicAccess& access) const {
 590   CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access();
 591   GraphKit* kit = access.kit();
 592   PhaseGVN& gvn = kit->gvn();
 593   Compile* C = Compile::current();
 594 
 595   Node* in_ctrl     = cmpx->in(MemNode::Control);
 596   Node* in_mem      = cmpx->in(MemNode::Memory);
 597   Node* in_adr      = cmpx->in(MemNode::Address);
 598   Node* in_val      = cmpx->in(MemNode::ValueIn);
 599   Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn);
 600 
 601   float likely                   = PROB_LIKELY(0.999);
 602 
 603   const TypePtr *adr_type        = cmpx->get_ptr_type();
 604   Compile::AliasType* alias_type = C->alias_type(adr_type);
 605   int alias_idx                  = C->get_alias_index(adr_type);
 606 
 607   // Outer check - true: continue, false: load and check
 608   Node* region  = new RegionNode(3);
 609   Node* phi     = new PhiNode(region, adr_type);
 610 
 611   // Inner check - is the healed ref equal to the expected
 612   Node* region2 = new RegionNode(3);
 613   Node* phi2    = new PhiNode(region2, adr_type);
 614 
 615   // Check if cmpx succeeded
 616   Node* cmp     = gvn.transform(new CmpPNode(cmpx, in_expected));
 617   Node* bol     = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool();
 618   IfNode* iff   = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
 619   Node* then    = gvn.transform(new IfTrueNode(iff));
 620   Node* elsen   = gvn.transform(new IfFalseNode(iff));
 621 
 622   Node* scmemproj1  = gvn.transform(new SCMemProjNode(cmpx));
 623   kit->set_memory(scmemproj1, alias_idx);
 624 
 625   // CAS fail - reload and heal oop
 626   Node* reload      = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
 627   Node* barrier     = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
 628   Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
 629   Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
 630 
 631   // Check load
 632   Node* tmpX    = gvn.transform(new CastP2XNode(NULL, barrierdata));
 633   Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
 634   Node* cmp2    = gvn.transform(new CmpXNode(tmpX, in_expX));
 635   Node *bol2    = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
 636   IfNode* iff2  = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
 637   Node* then2   = gvn.transform(new IfTrueNode(iff2));
 638   Node* elsen2  = gvn.transform(new IfFalseNode(iff2));
 639 
 640   // Redo CAS
 641   Node* cmpx2      = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order()));
 642   Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2));
 643   kit->set_control(elsen2);
 644   kit->set_memory(scmemproj2, alias_idx);
 645 
 646   // Merge inner flow - check if healed oop was equal too expected.
 647   region2->set_req(1, kit->control());
 648   region2->set_req(2, then2);
 649   phi2->set_req(1, cmpx2);
 650   phi2->set_req(2, barrierdata);
 651 
 652   // Merge outer flow - then check if first cas succeeded
 653   region->set_req(1, then);
 654   region->set_req(2, region2);
 655   phi->set_req(1, cmpx);
 656   phi->set_req(2, phi2);
 657 
 658   gvn.transform(region2);
 659   gvn.transform(phi2);
 660   gvn.transform(region);
 661   gvn.transform(phi);
 662 
 663   kit->set_control(region);
 664   kit->set_memory(in_mem, alias_idx);
 665   kit->insert_mem_bar(Op_MemBarCPUOrder);
 666 
 667   return phi;
 668 }
 669 
 670 Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const {
 671   PhaseGVN& gvn = kit->gvn();
 672   Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed);
 673   Node* transformed_barrier = gvn.transform(barrier);
 674 
 675   if (transformed_barrier->is_LoadBarrier()) {
 676     if (barrier == transformed_barrier) {
 677       kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)));
 678     }
 679     Node* result = gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop));
 680     assert(is_gc_barrier_node(result), "sanity");
 681     assert(step_over_gc_barrier(result) == val, "sanity");
 682     return result;
 683   } else {
 684     return val;
 685   }
 686 }
 687 
 688 static bool barrier_needed(C2Access access) {
 689   return ZBarrierSet::barrier_needed(access.decorators(), access.type());
 690 }
 691 
 692 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
 693   Node* p = BarrierSetC2::load_at_resolved(access, val_type);
 694   if (!barrier_needed(access)) {
 695     return p;
 696   }
 697 
 698   bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
 699 
 700   GraphKit* kit = access.kit();
 701   PhaseGVN& gvn = kit->gvn();
 702   Node* adr = access.addr().node();
 703   Node* heap_base_oop = access.base();
 704   bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0;
 705   if (unsafe) {
 706     if (!ZVerifyLoadBarriers) {
 707       p = load_barrier(kit, p, adr);
 708     } else {
 709       if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) {
 710         p = load_barrier(kit, p, adr);
 711       } else {
 712         IdealKit ideal(kit);
 713         IdealVariable res(ideal);
 714 #define __ ideal.
 715         __ declarations_done();
 716         __ set(res, p);
 717         __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); {
 718           kit->sync_kit(ideal);
 719           p = load_barrier(kit, p, adr);
 720           __ set(res, p);
 721           __ sync_kit(kit);
 722         } __ end_if();
 723         kit->final_sync(ideal);
 724         p = __ value(res);
 725 #undef __
 726       }
 727     }
 728     return p;
 729   } else {
 730     return load_barrier(access.kit(), p, access.addr().node(), weak, true, true);
 731   }
 732 }
 733 
 734 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
 735                                                     Node* new_val, const Type* val_type) const {
 736   Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
 737   if (!barrier_needed(access)) {
 738     return result;
 739   }
 740 
 741   access.set_needs_pinning(false);
 742   return make_cmpx_loadbarrier(access);
 743 }
 744 
 745 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
 746                                                      Node* new_val, const Type* value_type) const {
 747   Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
 748   if (!barrier_needed(access)) {
 749     return result;
 750   }
 751 
 752   Node* load_store = access.raw_access();
 753   bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0;
 754   bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR);
 755 
 756   if (!expected_is_null) {
 757     if (weak_cas) {
 758       access.set_needs_pinning(false);
 759       load_store = make_cas_loadbarrier(access);
 760     } else {
 761       access.set_needs_pinning(false);
 762       load_store = make_cas_loadbarrier(access);
 763     }
 764   }
 765 
 766   return load_store;
 767 }
 768 
 769 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const {
 770   Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
 771   if (!barrier_needed(access)) {
 772     return result;
 773   }
 774 
 775   Node* load_store = access.raw_access();
 776   Node* adr = access.addr().node();
 777 
 778   return load_barrier(access.kit(), load_store, adr, false, false, false);
 779 }
 780 
 781 // == Macro Expansion ==
 782 
 783 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {
 784   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
 785   Node* in_mem  = barrier->in(LoadBarrierNode::Memory);
 786   Node* in_val  = barrier->in(LoadBarrierNode::Oop);
 787   Node* in_adr  = barrier->in(LoadBarrierNode::Address);
 788 
 789   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
 790   Node* out_res  = barrier->proj_out(LoadBarrierNode::Oop);
 791 
 792   PhaseIterGVN &igvn = phase->igvn();
 793 
 794   if (ZVerifyLoadBarriers) {
 795     igvn.replace_node(out_res, in_val);
 796     igvn.replace_node(out_ctrl, in_ctrl);
 797     return;
 798   }
 799 
 800   if (barrier->can_be_eliminated()) {
 801     // Clone and pin the load for this barrier below the dominating
 802     // barrier: the load cannot be allowed to float above the
 803     // dominating barrier
 804     Node* load = in_val;
 805 
 806     if (load->is_Load()) {
 807       Node* new_load = load->clone();
 808       Node* addp = new_load->in(MemNode::Address);
 809       assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address");
 810       Node* cast = new CastPPNode(addp, igvn.type(addp), true);
 811       Node* ctrl = NULL;
 812       Node* similar = barrier->in(LoadBarrierNode::Similar);
 813       if (similar->is_Phi()) {
 814         // already expanded
 815         ctrl = similar->in(0);
 816       } else {
 817         assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape");
 818         ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control);
 819       }
 820       assert(ctrl != NULL, "bad control");
 821       cast->set_req(0, ctrl);
 822       igvn.transform(cast);
 823       new_load->set_req(MemNode::Address, cast);
 824       igvn.transform(new_load);
 825 
 826       igvn.replace_node(out_res, new_load);
 827       igvn.replace_node(out_ctrl, in_ctrl);
 828       return;
 829     }
 830     // cannot eliminate
 831   }
 832 
 833   // There are two cases that require the basic loadbarrier
 834   // 1) When the writeback of a healed oop must be avoided (swap)
 835   // 2) When we must guarantee that no reload of is done (swap, cas, cmpx)
 836   if (!barrier->is_writeback()) {
 837     assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop");
 838   }
 839 
 840   if (!barrier->oop_reload_allowed()) {
 841     expand_loadbarrier_basic(phase, barrier);
 842   } else {
 843     expand_loadbarrier_optimized(phase, barrier);
 844   }
 845 }
 846 
 847 // Basic loadbarrier using conventional argument passing
 848 void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
 849   PhaseIterGVN &igvn = phase->igvn();
 850 
 851   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
 852   Node* in_mem  = barrier->in(LoadBarrierNode::Memory);
 853   Node* in_val  = barrier->in(LoadBarrierNode::Oop);
 854   Node* in_adr  = barrier->in(LoadBarrierNode::Address);
 855 
 856   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
 857   Node* out_res  = barrier->proj_out(LoadBarrierNode::Oop);
 858 
 859   float unlikely  = PROB_UNLIKELY(0.999);
 860   const Type* in_val_maybe_null_t = igvn.type(in_val);
 861 
 862   Node* jthread = igvn.transform(new ThreadLocalNode());
 863   Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
 864   Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
 865   Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
 866   Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
 867   Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
 868   Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
 869   IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
 870   Node* then = igvn.transform(new IfTrueNode(iff));
 871   Node* elsen = igvn.transform(new IfFalseNode(iff));
 872 
 873   Node* result_region;
 874   Node* result_val;
 875 
 876   result_region = new RegionNode(3);
 877   result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);
 878 
 879   result_region->set_req(1, elsen);
 880   Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));
 881   res->init_req(0, elsen);
 882   result_val->set_req(1, res);
 883 
 884   const TypeFunc *tf = load_barrier_Type();
 885   Node* call;
 886   if (barrier->is_weak()) {
 887     call = new CallLeafNode(tf,
 888                             ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(),
 889                             "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded",
 890                             TypeRawPtr::BOTTOM);
 891   } else {
 892     call = new CallLeafNode(tf,
 893                             ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(),
 894                             "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded",
 895                             TypeRawPtr::BOTTOM);
 896   }
 897 
 898   call->init_req(TypeFunc::Control, then);
 899   call->init_req(TypeFunc::I_O    , phase->top());
 900   call->init_req(TypeFunc::Memory , in_mem);
 901   call->init_req(TypeFunc::FramePtr, phase->top());
 902   call->init_req(TypeFunc::ReturnAdr, phase->top());
 903   call->init_req(TypeFunc::Parms+0, in_val);
 904   if (barrier->is_writeback()) {
 905     call->init_req(TypeFunc::Parms+1, in_adr);
 906   } else {
 907     // When slow path is called with a null address, the healed oop will not be written back
 908     call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT));
 909   }
 910   call = igvn.transform(call);
 911 
 912   Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control));
 913   res = igvn.transform(new ProjNode(call, TypeFunc::Parms));
 914   res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t));
 915 
 916   result_region->set_req(2, ctrl);
 917   result_val->set_req(2, res);
 918 
 919   result_region = igvn.transform(result_region);
 920   result_val = igvn.transform(result_val);
 921 
 922   if (out_ctrl != NULL) { // Added if cond
 923     igvn.replace_node(out_ctrl, result_region);
 924   }
 925   igvn.replace_node(out_res, result_val);
 926 }
 927 
 928 // Optimized, low spill, loadbarrier variant using stub specialized on register used
 929 void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
 930   PhaseIterGVN &igvn = phase->igvn();
 931 #ifdef PRINT_NODE_TRAVERSALS
 932   Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop);
 933 #endif
 934 
 935   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
 936   Node* in_mem = barrier->in(LoadBarrierNode::Memory);
 937   Node* in_val = barrier->in(LoadBarrierNode::Oop);
 938   Node* in_adr = barrier->in(LoadBarrierNode::Address);
 939 
 940   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
 941   Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
 942 
 943   assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
 944 
 945 #ifdef PRINT_NODE_TRAVERSALS
 946   tty->print("\n\n\nBefore barrier optimization:\n");
 947   traverse(barrier, out_ctrl, out_res, -1);
 948 
 949   tty->print("\nBefore barrier optimization:  preceding_barrier_node\n");
 950   traverse(preceding_barrier_node, out_ctrl, out_res, -1);
 951 #endif
 952 
 953   float unlikely  = PROB_UNLIKELY(0.999);
 954 
 955   Node* jthread = igvn.transform(new ThreadLocalNode());
 956   Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
 957   Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
 958                                                  TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
 959                                                  MemNode::unordered));
 960   Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
 961   Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
 962   Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
 963   Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
 964   IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
 965   Node* then = igvn.transform(new IfTrueNode(iff));
 966   Node* elsen = igvn.transform(new IfFalseNode(iff));
 967 
 968   Node* slow_path_surrogate;
 969   if (!barrier->is_weak()) {
 970     slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
 971                                                                     (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
 972   } else {
 973     slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
 974                                                                         (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
 975   }
 976 
 977   Node *new_loadp;
 978   new_loadp = slow_path_surrogate;
 979   // Create the final region/phi pair to converge cntl/data paths to downstream code
 980   Node* result_region = igvn.transform(new RegionNode(3));
 981   result_region->set_req(1, then);
 982   result_region->set_req(2, elsen);
 983 
 984   Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
 985   result_phi->set_req(1, new_loadp);
 986   result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
 987 
 988   // Finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution
 989   // igvn.replace_node(out_ctrl, result_region);
 990   if (out_ctrl != NULL) { // added if cond
 991     igvn.replace_node(out_ctrl, result_region);
 992   }
 993   igvn.replace_node(out_res, result_phi);
 994 
 995   assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
 996 
 997 #ifdef PRINT_NODE_TRAVERSALS
 998   tty->print("\nAfter barrier optimization:  old out_ctrl\n");
 999   traverse(out_ctrl, out_ctrl, out_res, -1);
1000   tty->print("\nAfter barrier optimization:  old out_res\n");
1001   traverse(out_res, out_ctrl, out_res, -1);
1002   tty->print("\nAfter barrier optimization:  old barrier\n");
1003   traverse(barrier, out_ctrl, out_res, -1);
1004   tty->print("\nAfter barrier optimization:  preceding_barrier_node\n");
1005   traverse(preceding_barrier_node, result_region, result_phi, -1);
1006 #endif
1007 
1008   assert(is_gc_barrier_node(result_phi), "sanity");
1009   assert(step_over_gc_barrier(result_phi) == in_val, "sanity");
1010 
1011   return;
1012 }
1013 
1014 bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const {
1015   Compile* C = Compile::current();
1016   PhaseIterGVN &igvn = macro->igvn();
1017   ZBarrierSetC2State* s = state();
1018   if (s->load_barrier_count() > 0) {
1019 #ifdef ASSERT
1020     verify_gc_barriers(false);
1021 #endif
1022     igvn.set_delay_transform(true);
1023     int skipped = 0;
1024     while (s->load_barrier_count() > skipped) {
1025       int load_barrier_count = s->load_barrier_count();
1026       LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
1027       if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
1028         // Node is unreachable, so don't try to expand it
1029         s->remove_load_barrier_node(n);
1030         continue;
1031       }
1032       if (!n->can_be_eliminated()) {
1033         skipped++;
1034         continue;
1035       }
1036       expand_loadbarrier_node(macro, n);
1037       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1038       if (C->failing())  return true;
1039     }
1040     while (s->load_barrier_count() > 0) {
1041       int load_barrier_count = s->load_barrier_count();
1042       LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);
1043       assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");
1044       assert(!n->can_be_eliminated(), "should have been processed already");
1045       expand_loadbarrier_node(macro, n);
1046       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1047       if (C->failing())  return true;
1048     }
1049     igvn.set_delay_transform(false);
1050     igvn.optimize();
1051     if (C->failing())  return true;
1052   }
1053   return false;
1054 }
1055 
1056 // == Loop optimization ==
1057 
1058 static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1059   PhaseIterGVN &igvn = phase->igvn();
1060   Compile* C = Compile::current();
1061 
1062   LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round);
1063   if (lb2 != NULL) {
1064     if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) {
1065       assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), "");
1066       igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop));
1067       C->set_major_progress();
1068     } else  {
1069       // That transformation may cause the Similar edge on dominated load barriers to be invalid
1070       lb->fix_similar_in_uses(&igvn);
1071 
1072       Node* val = lb->proj_out(LoadBarrierNode::Oop);
1073       assert(lb2->has_true_uses(), "");
1074       assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), "");
1075 
1076       phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1077       phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1078       igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop));
1079 
1080       return true;
1081     }
1082   }
1083   return false;
1084 }
1085 
1086 static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) {
1087   assert(dom->is_Region() || i == -1, "");
1088   Node* m = mem;
1089   while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) {
1090     if (m->is_Mem()) {
1091       assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, "");
1092       m = m->in(MemNode::Memory);
1093     } else if (m->is_MergeMem()) {
1094       m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1095     } else if (m->is_Phi()) {
1096       if (m->in(0) == dom && i != -1) {
1097         m = m->in(i);
1098         break;
1099       } else {
1100         m = m->in(LoopNode::EntryControl);
1101       }
1102     } else if (m->is_Proj()) {
1103       m = m->in(0);
1104     } else if (m->is_SafePoint() || m->is_MemBar()) {
1105       m = m->in(TypeFunc::Memory);
1106     } else {
1107 #ifdef ASSERT
1108       m->dump();
1109 #endif
1110       ShouldNotReachHere();
1111     }
1112   }
1113   return m;
1114 }
1115 
1116 static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) {
1117   PhaseIterGVN &igvn = phase->igvn();
1118   Compile* C = Compile::current();
1119   Node* the_clone = lb->clone();
1120   the_clone->set_req(LoadBarrierNode::Control, ctl);
1121   the_clone->set_req(LoadBarrierNode::Memory, mem);
1122   if (oop_in != NULL) {
1123     the_clone->set_req(LoadBarrierNode::Oop, oop_in);
1124   }
1125 
1126   LoadBarrierNode* new_lb = the_clone->as_LoadBarrier();
1127   igvn.register_new_node_with_optimizer(new_lb);
1128   IdealLoopTree *loop = phase->get_loop(new_lb->in(0));
1129   phase->set_ctrl(new_lb, new_lb->in(0));
1130   phase->set_loop(new_lb, loop);
1131   phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1);
1132   if (!loop->_child) {
1133     loop->_body.push(new_lb);
1134   }
1135 
1136   Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control);
1137   igvn.register_new_node_with_optimizer(proj_ctl);
1138   phase->set_ctrl(proj_ctl, proj_ctl->in(0));
1139   phase->set_loop(proj_ctl, loop);
1140   phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1);
1141   if (!loop->_child) {
1142     loop->_body.push(proj_ctl);
1143   }
1144 
1145   Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop);
1146   phase->register_new_node(proj_oop, new_lb);
1147 
1148   if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) {
1149     LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier();
1150     if (!phase->is_dominator(similar, ctl)) {
1151       igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top());
1152     }
1153   }
1154 
1155   return new_lb;
1156 }
1157 
1158 static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) {
1159   PhaseIterGVN &igvn = phase->igvn();
1160   Node* val = lb->proj_out(LoadBarrierNode::Oop);
1161   igvn.replace_node(val, new_val);
1162   phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1163   phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1164 }
1165 
1166 static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1167   PhaseIterGVN &igvn = phase->igvn();
1168   Compile* C = Compile::current();
1169 
1170   if (lb->in(LoadBarrierNode::Oop)->is_Phi()) {
1171     Node* oop_phi = lb->in(LoadBarrierNode::Oop);
1172 
1173     if ((oop_phi->req() != 3) || (oop_phi->in(2) == oop_phi)) {
1174       // Ignore phis with only one input
1175       return false;
1176     }
1177 
1178     if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)),
1179                             oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) {
1180       // That transformation may cause the Similar edge on dominated load barriers to be invalid
1181       lb->fix_similar_in_uses(&igvn);
1182 
1183       RegionNode* region = oop_phi->in(0)->as_Region();
1184 
1185       int backedge = LoopNode::LoopBackControl;
1186       if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) {
1187         Node* c = region->in(backedge)->in(0)->in(0);
1188         assert(c->unique_ctrl_out() == region->in(backedge)->in(0), "");
1189         Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge);
1190         Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop;
1191         if (!phase->is_dominator(oop_c, c)) {
1192           return false;
1193         }
1194       }
1195 
1196       // If the node on the backedge above the phi is the node itself - we have a self loop.
1197       // Don't clone - this will be folded later.
1198       if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) {
1199         return false;
1200       }
1201 
1202       bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined();
1203       Node *phi = oop_phi->clone();
1204 
1205       for (uint i = 1; i < region->req(); i++) {
1206         Node* ctrl = region->in(i);
1207         if (ctrl != C->top()) {
1208           assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), "");
1209 
1210           Node* mem = lb->in(LoadBarrierNode::Memory);
1211           Node* m = find_dominating_memory(phase, mem, region, i);
1212 
1213           if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) {
1214             ctrl = ctrl->in(0)->in(0);
1215           } else if (region->is_Loop() && is_strip_mined) {
1216             // If this is a strip mined loop, control must move above OuterStripMinedLoop
1217             assert(i == LoopNode::EntryControl, "check");
1218             assert(ctrl->is_OuterStripMinedLoop(), "sanity");
1219             ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl);
1220           }
1221 
1222           LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i));
1223           Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control);
1224 
1225           if (is_strip_mined && (i == LoopNode::EntryControl)) {
1226             assert(region->in(i)->is_OuterStripMinedLoop(), "");
1227             igvn.replace_input_of(region->in(i), i, out_ctrl);
1228             phase->set_idom(region->in(i), out_ctrl, phase->dom_depth(out_ctrl));
1229           } else if (ctrl == region->in(i)) {
1230             igvn.replace_input_of(region, i, out_ctrl);
1231             // Only update the idom if is the loop entry we are updating
1232             // - A loop backedge doesn't change the idom
1233             if (region->is_Loop() && i == LoopNode::EntryControl) {
1234               phase->set_idom(region, out_ctrl, phase->dom_depth(out_ctrl));
1235             }
1236           } else {
1237             Node* iff = region->in(i)->in(0);
1238             igvn.replace_input_of(iff, 0, out_ctrl);
1239             phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1);
1240           }
1241           phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop));
1242         }
1243       }
1244       phase->register_new_node(phi, region);
1245       replace_barrier(phase, lb, phi);
1246 
1247       if (region->is_Loop()) {
1248         // Load barrier moved to the back edge of the Loop may now
1249         // have a safepoint on the path to the barrier on the Similar
1250         // edge
1251         igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top());
1252         Node* head = region->in(LoopNode::EntryControl);
1253         phase->set_idom(region, head, phase->dom_depth(head)+1);
1254         phase->recompute_dom_depth();
1255         if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) {
1256           head->as_CountedLoop()->set_normal_loop();
1257         }
1258       }
1259 
1260       return true;
1261     }
1262   }
1263 
1264   return false;
1265 }
1266 
1267 static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1268   PhaseIterGVN &igvn = phase->igvn();
1269   IdealLoopTree *lb_loop = phase->get_loop(lb->in(0));
1270   if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) {
1271     Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop));
1272     IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl);
1273     IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address)));
1274     if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) {
1275       // That transformation may cause the Similar edge on dominated load barriers to be invalid
1276       lb->fix_similar_in_uses(&igvn);
1277 
1278       Node* head = lb_loop->_head;
1279       assert(head->is_Loop(), "");
1280 
1281       if (phase->is_dominator(head, oop_ctrl)) {
1282         assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, "");
1283         assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), "");
1284         return false;
1285       }
1286 
1287       if (head->is_CountedLoop()) {
1288         CountedLoopNode* cloop = head->as_CountedLoop();
1289         if (cloop->is_main_loop()) {
1290           cloop->set_normal_loop();
1291         }
1292         // When we are moving barrier out of a counted loop,
1293         // make sure we move it all the way out of the strip mined outer loop.
1294         if (cloop->is_strip_mined()) {
1295           head = cloop->outer_loop();
1296         }
1297       }
1298 
1299       Node* mem = lb->in(LoadBarrierNode::Memory);
1300       Node* m = find_dominating_memory(phase, mem, head, -1);
1301 
1302       LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL);
1303 
1304       assert(phase->idom(head) == head->in(LoopNode::EntryControl), "");
1305       Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1306       igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl);
1307       phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1);
1308 
1309       replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop));
1310 
1311       phase->recompute_dom_depth();
1312 
1313       return true;
1314     }
1315   }
1316 
1317   return false;
1318 }
1319 
1320 static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1321   PhaseIterGVN &igvn = phase->igvn();
1322   Node* in_val = lb->in(LoadBarrierNode::Oop);
1323   for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) {
1324     Node* u = in_val->fast_out(i);
1325     if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
1326       Node* this_ctrl = lb->in(LoadBarrierNode::Control);
1327       Node* other_ctrl = u->in(LoadBarrierNode::Control);
1328 
1329       Node* lca = phase->dom_lca(this_ctrl, other_ctrl);
1330       Node* proj1 = NULL;
1331       Node* proj2 = NULL;
1332       bool ok = (lb->in(LoadBarrierNode::Address) == u->in(LoadBarrierNode::Address));
1333 
1334       while (this_ctrl != lca && ok) {
1335         if (this_ctrl->in(0) != NULL &&
1336             this_ctrl->in(0)->is_MultiBranch()) {
1337           if (this_ctrl->in(0)->in(0) == lca) {
1338             assert(proj1 == NULL, "");
1339             assert(this_ctrl->is_Proj(), "");
1340             proj1 = this_ctrl;
1341           } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1342             ok = false;
1343           }
1344         }
1345         this_ctrl = phase->idom(this_ctrl);
1346       }
1347       while (other_ctrl != lca && ok) {
1348         if (other_ctrl->in(0) != NULL &&
1349             other_ctrl->in(0)->is_MultiBranch()) {
1350           if (other_ctrl->in(0)->in(0) == lca) {
1351             assert(other_ctrl->is_Proj(), "");
1352             assert(proj2 == NULL, "");
1353             proj2 = other_ctrl;
1354           } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1355             ok = false;
1356           }
1357         }
1358         other_ctrl = phase->idom(other_ctrl);
1359       }
1360       assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), "");
1361       if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) {
1362         // That transformation may cause the Similar edge on dominated load barriers to be invalid
1363         lb->fix_similar_in_uses(&igvn);
1364         u->as_LoadBarrier()->fix_similar_in_uses(&igvn);
1365 
1366         Node* split = lca->unique_ctrl_out();
1367         assert(split->in(0) == lca, "");
1368 
1369         Node* mem = lb->in(LoadBarrierNode::Memory);
1370         Node* m = find_dominating_memory(phase, mem, split, -1);
1371         LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL);
1372 
1373         Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1374         igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control));
1375         phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1);
1376 
1377         Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop);
1378         replace_barrier(phase, lb, proj_oop);
1379         replace_barrier(phase, u->as_LoadBarrier(), proj_oop);
1380 
1381         phase->recompute_dom_depth();
1382 
1383         return true;
1384       }
1385     }
1386   }
1387 
1388   return false;
1389 }
1390 
1391 static void optimize_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1392   Compile* C = Compile::current();
1393 
1394   if (!C->directive()->ZOptimizeLoadBarriersOption) {
1395     return;
1396   }
1397 
1398   if (lb->has_true_uses()) {
1399     if (replace_with_dominating_barrier(phase, lb, last_round)) {
1400       return;
1401     }
1402 
1403     if (split_barrier_thru_phi(phase, lb)) {
1404       return;
1405     }
1406 
1407     if (move_out_of_loop(phase, lb)) {
1408       return;
1409     }
1410 
1411     if (common_barriers(phase, lb)) {
1412       return;
1413     }
1414   }
1415 }
1416 
1417 void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) {
1418   if (node->is_LoadBarrier()) {
1419     optimize_load_barrier(phase, node->as_LoadBarrier(), last_round);
1420   }
1421 }
1422 
1423 Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const {
1424   Node* node = c;
1425 
1426   // 1. This step follows potential oop projections of a load barrier before expansion
1427   if (node->is_Proj()) {
1428     node = node->in(0);
1429   }
1430 
1431   // 2. This step checks for unexpanded load barriers
1432   if (node->is_LoadBarrier()) {
1433     return node->in(LoadBarrierNode::Oop);
1434   }
1435 
1436   // 3. This step checks for the phi corresponding to an optimized load barrier expansion
1437   if (node->is_Phi()) {
1438     PhiNode* phi = node->as_Phi();
1439     Node* n = phi->in(1);
1440     if (n != NULL && (n->is_LoadBarrierSlowReg() ||  n->is_LoadBarrierWeakSlowReg())) {
1441       assert(c == node, "projections from step 1 should only be seen before macro expansion");
1442       return phi->in(2);
1443     }
1444   }
1445 
1446   return c;
1447 }
1448 
1449 // == Verification ==
1450 
1451 #ifdef ASSERT
1452 
1453 static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
1454   if (visited.test_set(n->_idx)) {
1455     return true;
1456   }
1457 
1458   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1459     Node* u = n->fast_out(i);
1460     if (u->is_LoadBarrier()) {
1461     } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) {
1462       if (!look_for_barrier(u, post_parse, visited)) {
1463         return false;
1464       }
1465     } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) {
1466       if (!look_for_barrier(u, post_parse, visited)) {
1467         return false;
1468       }
1469     } else if (u->Opcode() != Op_SCMemProj) {
1470       tty->print("bad use"); u->dump();
1471       return false;
1472     }
1473   }
1474 
1475   return true;
1476 }
1477 
1478 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
1479   ZBarrierSetC2State* s = state();
1480   Compile* C = Compile::current();
1481   ResourceMark rm;
1482   VectorSet visited(Thread::current()->resource_area());
1483   for (int i = 0; i < s->load_barrier_count(); i++) {
1484     LoadBarrierNode* n = s->load_barrier_node(i);
1485 
1486     // The dominating barrier on the same address if it exists and
1487     // this barrier must not be applied on the value from the same
1488     // load otherwise the value is not reloaded before it's used the
1489     // second time.
1490     assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1491            (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1492             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
1493             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
1494            "broken similar edge");
1495 
1496     assert(post_parse || n->as_LoadBarrier()->has_true_uses(),
1497            "found unneeded load barrier");
1498 
1499     // Several load barrier nodes chained through their Similar edge
1500     // break the code that remove the barriers in final graph reshape.
1501     assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1502            (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1503             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),
1504            "chain of Similar load barriers");
1505 
1506     if (!n->in(LoadBarrierNode::Similar)->is_top()) {
1507       ResourceMark rm;
1508       Unique_Node_List wq;
1509       Node* other = n->in(LoadBarrierNode::Similar)->in(0);
1510       wq.push(n);
1511       bool ok = true;
1512       bool dom_found = false;
1513       for (uint next = 0; next < wq.size(); ++next) {
1514         Node *n = wq.at(next);
1515         assert(n->is_CFG(), "");
1516         assert(!n->is_SafePoint(), "");
1517 
1518         if (n == other) {
1519           continue;
1520         }
1521 
1522         if (n->is_Region()) {
1523           for (uint i = 1; i < n->req(); i++) {
1524             Node* m = n->in(i);
1525             if (m != NULL) {
1526               wq.push(m);
1527             }
1528           }
1529         } else {
1530           Node* m = n->in(0);
1531           if (m != NULL) {
1532             wq.push(m);
1533           }
1534         }
1535       }
1536     }
1537 
1538     if (ZVerifyLoadBarriers) {
1539       if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) {
1540         visited.Clear();
1541         bool found = look_for_barrier(n, post_parse, visited);
1542         if (!found) {
1543           n->dump(1);
1544           n->dump(-3);
1545           stringStream ss;
1546           C->method()->print_short_name(&ss);
1547           tty->print_cr("-%s-", ss.as_string());
1548           assert(found, "");
1549         }
1550       }
1551     }
1552   }
1553 }
1554 
1555 #endif