< prev index next >

src/hotspot/share/opto/macro.cpp

Print this page




  30 #include "opto/arraycopynode.hpp"
  31 #include "opto/callnode.hpp"
  32 #include "opto/castnode.hpp"
  33 #include "opto/cfgnode.hpp"
  34 #include "opto/compile.hpp"
  35 #include "opto/convertnode.hpp"
  36 #include "opto/graphKit.hpp"
  37 #include "opto/locknode.hpp"
  38 #include "opto/loopnode.hpp"
  39 #include "opto/macro.hpp"
  40 #include "opto/memnode.hpp"
  41 #include "opto/narrowptrnode.hpp"
  42 #include "opto/node.hpp"
  43 #include "opto/opaquenode.hpp"
  44 #include "opto/phaseX.hpp"
  45 #include "opto/rootnode.hpp"
  46 #include "opto/runtime.hpp"
  47 #include "opto/subnode.hpp"
  48 #include "opto/type.hpp"
  49 #include "runtime/sharedRuntime.hpp"

  50 #if INCLUDE_G1GC
  51 #include "gc/g1/g1ThreadLocalData.hpp"
  52 #endif // INCLUDE_G1GC



  53 
  54 
  55 //
  56 // Replace any references to "oldref" in inputs to "use" with "newref".
  57 // Returns the number of replacements made.
  58 //
  59 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
  60   int nreplacements = 0;
  61   uint req = use->req();
  62   for (uint j = 0; j < use->len(); j++) {
  63     Node *uin = use->in(j);
  64     if (uin == oldref) {
  65       if (j < req)
  66         use->set_req(j, newref);
  67       else
  68         use->set_prec(j, newref);
  69       nreplacements++;
  70     } else if (j >= req && uin == NULL) {
  71       break;
  72     }


 417   for (uint j = 1; j < length; j++) {
 418     Node *in = mem->in(j);
 419     if (in == NULL || in->is_top()) {
 420       values.at_put(j, in);
 421     } else  {
 422       Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
 423       if (val == start_mem || val == alloc_mem) {
 424         // hit a sentinel, return appropriate 0 value
 425         values.at_put(j, _igvn.zerocon(ft));
 426         continue;
 427       }
 428       if (val->is_Initialize()) {
 429         val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
 430       }
 431       if (val == NULL) {
 432         return NULL;  // can't find a value on this path
 433       }
 434       if (val == mem) {
 435         values.at_put(j, mem);
 436       } else if (val->is_Store()) {
 437         values.at_put(j, val->in(MemNode::ValueIn));







 438       } else if(val->is_Proj() && val->in(0) == alloc) {
 439         values.at_put(j, _igvn.zerocon(ft));
 440       } else if (val->is_Phi()) {
 441         val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1);
 442         if (val == NULL) {
 443           return NULL;
 444         }
 445         values.at_put(j, val);
 446       } else if (val->Opcode() == Op_SCMemProj) {
 447         assert(val->in(0)->is_LoadStore() ||
 448                val->in(0)->Opcode() == Op_EncodeISOArray ||
 449                val->in(0)->Opcode() == Op_StrCompressedCopy, "sanity");
 450         assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field");
 451         return NULL;
 452       } else if (val->is_ArrayCopy()) {
 453         Node* res = make_arraycopy_load(val->as_ArrayCopy(), offset, val->in(0), val->in(TypeFunc::Memory), ft, phi_type, alloc);
 454         if (res == NULL) {
 455           return NULL;
 456         }
 457         values.at_put(j, res);


 529           unique_input = top;
 530           break;
 531         }
 532       }
 533       if (unique_input != NULL && unique_input != top) {
 534         mem = unique_input;
 535       } else {
 536         done = true;
 537       }
 538     } else if (mem->is_ArrayCopy()) {
 539       done = true;
 540     } else {
 541       assert(false, "unexpected node");
 542     }
 543   }
 544   if (mem != NULL) {
 545     if (mem == start_mem || mem == alloc_mem) {
 546       // hit a sentinel, return appropriate 0 value
 547       return _igvn.zerocon(ft);
 548     } else if (mem->is_Store()) {
 549       return mem->in(MemNode::ValueIn);







 550     } else if (mem->is_Phi()) {
 551       // attempt to produce a Phi reflecting the values on the input paths of the Phi
 552       Node_Stack value_phis(a, 8);
 553       Node * phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit);
 554       if (phi != NULL) {
 555         return phi;
 556       } else {
 557         // Kill all new Phis
 558         while(value_phis.is_nonempty()) {
 559           Node* n = value_phis.node();
 560           _igvn.replace_node(n, C->top());
 561           value_phis.pop();
 562         }
 563       }
 564     } else if (mem->is_ArrayCopy()) {
 565       Node* ctl = mem->in(0);
 566       Node* m = mem->in(TypeFunc::Memory);
 567       if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
 568         // pin the loads in the uncommon trap path
 569         ctl = sfpt_ctl;


 606   }
 607 
 608   if (can_eliminate && res != NULL) {
 609     for (DUIterator_Fast jmax, j = res->fast_outs(jmax);
 610                                j < jmax && can_eliminate; j++) {
 611       Node* use = res->fast_out(j);
 612 
 613       if (use->is_AddP()) {
 614         const TypePtr* addp_type = _igvn.type(use)->is_ptr();
 615         int offset = addp_type->offset();
 616 
 617         if (offset == Type::OffsetTop || offset == Type::OffsetBot) {
 618           NOT_PRODUCT(fail_eliminate = "Undefined field referrence";)
 619           can_eliminate = false;
 620           break;
 621         }
 622         for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
 623                                    k < kmax && can_eliminate; k++) {
 624           Node* n = use->fast_out(k);
 625           if (!n->is_Store() && n->Opcode() != Op_CastP2X &&

 626               !(n->is_ArrayCopy() &&
 627                 n->as_ArrayCopy()->is_clonebasic() &&
 628                 n->in(ArrayCopyNode::Dest) == use)) {
 629             DEBUG_ONLY(disq_node = n;)
 630             if (n->is_Load() || n->is_LoadStore()) {
 631               NOT_PRODUCT(fail_eliminate = "Field load";)
 632             } else {
 633               NOT_PRODUCT(fail_eliminate = "Not store field referrence";)
 634             }
 635             can_eliminate = false;
 636           }
 637         }
 638       } else if (use->is_ArrayCopy() &&
 639                  (use->as_ArrayCopy()->is_arraycopy_validated() ||
 640                   use->as_ArrayCopy()->is_copyof_validated() ||
 641                   use->as_ArrayCopy()->is_copyofrange_validated()) &&
 642                  use->in(ArrayCopyNode::Dest) == res) {
 643         // ok to eliminate
 644       } else if (use->is_SafePoint()) {
 645         SafePointNode* sfpt = use->as_SafePoint();


 916                      "MemBarVolatile should be eliminated for non-escaping object");
 917             }
 918 #endif
 919             _igvn.replace_node(n, n->in(MemNode::Memory));
 920           } else if (n->is_ArrayCopy()) {
 921             // Disconnect ArrayCopy node
 922             ArrayCopyNode* ac = n->as_ArrayCopy();
 923             assert(ac->is_clonebasic(), "unexpected array copy kind");
 924             Node* membar_after = ac->proj_out(TypeFunc::Control)->unique_ctrl_out();
 925             disconnect_projections(ac, _igvn);
 926             assert(alloc->in(0)->is_Proj() && alloc->in(0)->in(0)->Opcode() == Op_MemBarCPUOrder, "mem barrier expected before allocation");
 927             Node* membar_before = alloc->in(0)->in(0);
 928             disconnect_projections(membar_before->as_MemBar(), _igvn);
 929             if (membar_after->is_MemBar()) {
 930               disconnect_projections(membar_after->as_MemBar(), _igvn);
 931             }
 932           } else {
 933             eliminate_gc_barrier(n);
 934           }
 935           k -= (oc2 - use->outcnt());



 936         }
 937       } else if (use->is_ArrayCopy()) {
 938         // Disconnect ArrayCopy node
 939         ArrayCopyNode* ac = use->as_ArrayCopy();
 940         assert(ac->is_arraycopy_validated() ||
 941                ac->is_copyof_validated() ||
 942                ac->is_copyofrange_validated(), "unsupported");
 943         CallProjections callprojs;
 944         ac->extract_projections(&callprojs, true);
 945 
 946         _igvn.replace_node(callprojs.fallthrough_ioproj, ac->in(TypeFunc::I_O));
 947         _igvn.replace_node(callprojs.fallthrough_memproj, ac->in(TypeFunc::Memory));
 948         _igvn.replace_node(callprojs.fallthrough_catchproj, ac->in(TypeFunc::Control));
 949 
 950         // Set control to top. IGVN will remove the remaining projections
 951         ac->set_req(0, top());
 952         ac->replace_edge(res, top());
 953 
 954         // Disconnect src right away: it can help find new
 955         // opportunities for allocation elimination




  30 #include "opto/arraycopynode.hpp"
  31 #include "opto/callnode.hpp"
  32 #include "opto/castnode.hpp"
  33 #include "opto/cfgnode.hpp"
  34 #include "opto/compile.hpp"
  35 #include "opto/convertnode.hpp"
  36 #include "opto/graphKit.hpp"
  37 #include "opto/locknode.hpp"
  38 #include "opto/loopnode.hpp"
  39 #include "opto/macro.hpp"
  40 #include "opto/memnode.hpp"
  41 #include "opto/narrowptrnode.hpp"
  42 #include "opto/node.hpp"
  43 #include "opto/opaquenode.hpp"
  44 #include "opto/phaseX.hpp"
  45 #include "opto/rootnode.hpp"
  46 #include "opto/runtime.hpp"
  47 #include "opto/subnode.hpp"
  48 #include "opto/type.hpp"
  49 #include "runtime/sharedRuntime.hpp"
  50 #include "utilities/macros.hpp"
  51 #if INCLUDE_G1GC
  52 #include "gc/g1/g1ThreadLocalData.hpp"
  53 #endif // INCLUDE_G1GC
  54 #if INCLUDE_SHENANDOAHGC
  55 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
  56 #endif
  57 
  58 
  59 //
  60 // Replace any references to "oldref" in inputs to "use" with "newref".
  61 // Returns the number of replacements made.
  62 //
  63 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
  64   int nreplacements = 0;
  65   uint req = use->req();
  66   for (uint j = 0; j < use->len(); j++) {
  67     Node *uin = use->in(j);
  68     if (uin == oldref) {
  69       if (j < req)
  70         use->set_req(j, newref);
  71       else
  72         use->set_prec(j, newref);
  73       nreplacements++;
  74     } else if (j >= req && uin == NULL) {
  75       break;
  76     }


 421   for (uint j = 1; j < length; j++) {
 422     Node *in = mem->in(j);
 423     if (in == NULL || in->is_top()) {
 424       values.at_put(j, in);
 425     } else  {
 426       Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
 427       if (val == start_mem || val == alloc_mem) {
 428         // hit a sentinel, return appropriate 0 value
 429         values.at_put(j, _igvn.zerocon(ft));
 430         continue;
 431       }
 432       if (val->is_Initialize()) {
 433         val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
 434       }
 435       if (val == NULL) {
 436         return NULL;  // can't find a value on this path
 437       }
 438       if (val == mem) {
 439         values.at_put(j, mem);
 440       } else if (val->is_Store()) {
 441         Node* n = val->in(MemNode::ValueIn);
 442 #if INCLUDE_SHENANDOAHGC
 443         if (UseShenandoahGC) {
 444           BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 445           n = bs->step_over_gc_barrier(n);
 446         }
 447 #endif
 448         values.at_put(j, n);
 449       } else if(val->is_Proj() && val->in(0) == alloc) {
 450         values.at_put(j, _igvn.zerocon(ft));
 451       } else if (val->is_Phi()) {
 452         val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1);
 453         if (val == NULL) {
 454           return NULL;
 455         }
 456         values.at_put(j, val);
 457       } else if (val->Opcode() == Op_SCMemProj) {
 458         assert(val->in(0)->is_LoadStore() ||
 459                val->in(0)->Opcode() == Op_EncodeISOArray ||
 460                val->in(0)->Opcode() == Op_StrCompressedCopy, "sanity");
 461         assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field");
 462         return NULL;
 463       } else if (val->is_ArrayCopy()) {
 464         Node* res = make_arraycopy_load(val->as_ArrayCopy(), offset, val->in(0), val->in(TypeFunc::Memory), ft, phi_type, alloc);
 465         if (res == NULL) {
 466           return NULL;
 467         }
 468         values.at_put(j, res);


 540           unique_input = top;
 541           break;
 542         }
 543       }
 544       if (unique_input != NULL && unique_input != top) {
 545         mem = unique_input;
 546       } else {
 547         done = true;
 548       }
 549     } else if (mem->is_ArrayCopy()) {
 550       done = true;
 551     } else {
 552       assert(false, "unexpected node");
 553     }
 554   }
 555   if (mem != NULL) {
 556     if (mem == start_mem || mem == alloc_mem) {
 557       // hit a sentinel, return appropriate 0 value
 558       return _igvn.zerocon(ft);
 559     } else if (mem->is_Store()) {
 560       Node* n = mem->in(MemNode::ValueIn);
 561 #if INCLUDE_SHENANDOAHGC
 562       if (UseShenandoahGC) {
 563         BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 564         n = bs->step_over_gc_barrier(n);
 565       }
 566 #endif
 567       return n;
 568     } else if (mem->is_Phi()) {
 569       // attempt to produce a Phi reflecting the values on the input paths of the Phi
 570       Node_Stack value_phis(a, 8);
 571       Node * phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit);
 572       if (phi != NULL) {
 573         return phi;
 574       } else {
 575         // Kill all new Phis
 576         while(value_phis.is_nonempty()) {
 577           Node* n = value_phis.node();
 578           _igvn.replace_node(n, C->top());
 579           value_phis.pop();
 580         }
 581       }
 582     } else if (mem->is_ArrayCopy()) {
 583       Node* ctl = mem->in(0);
 584       Node* m = mem->in(TypeFunc::Memory);
 585       if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
 586         // pin the loads in the uncommon trap path
 587         ctl = sfpt_ctl;


 624   }
 625 
 626   if (can_eliminate && res != NULL) {
 627     for (DUIterator_Fast jmax, j = res->fast_outs(jmax);
 628                                j < jmax && can_eliminate; j++) {
 629       Node* use = res->fast_out(j);
 630 
 631       if (use->is_AddP()) {
 632         const TypePtr* addp_type = _igvn.type(use)->is_ptr();
 633         int offset = addp_type->offset();
 634 
 635         if (offset == Type::OffsetTop || offset == Type::OffsetBot) {
 636           NOT_PRODUCT(fail_eliminate = "Undefined field referrence";)
 637           can_eliminate = false;
 638           break;
 639         }
 640         for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
 641                                    k < kmax && can_eliminate; k++) {
 642           Node* n = use->fast_out(k);
 643           if (!n->is_Store() && n->Opcode() != Op_CastP2X &&
 644               SHENANDOAHGC_ONLY((!UseShenandoahGC || !ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(n)) &&)
 645               !(n->is_ArrayCopy() &&
 646                 n->as_ArrayCopy()->is_clonebasic() &&
 647                 n->in(ArrayCopyNode::Dest) == use)) {
 648             DEBUG_ONLY(disq_node = n;)
 649             if (n->is_Load() || n->is_LoadStore()) {
 650               NOT_PRODUCT(fail_eliminate = "Field load";)
 651             } else {
 652               NOT_PRODUCT(fail_eliminate = "Not store field referrence";)
 653             }
 654             can_eliminate = false;
 655           }
 656         }
 657       } else if (use->is_ArrayCopy() &&
 658                  (use->as_ArrayCopy()->is_arraycopy_validated() ||
 659                   use->as_ArrayCopy()->is_copyof_validated() ||
 660                   use->as_ArrayCopy()->is_copyofrange_validated()) &&
 661                  use->in(ArrayCopyNode::Dest) == res) {
 662         // ok to eliminate
 663       } else if (use->is_SafePoint()) {
 664         SafePointNode* sfpt = use->as_SafePoint();


 935                      "MemBarVolatile should be eliminated for non-escaping object");
 936             }
 937 #endif
 938             _igvn.replace_node(n, n->in(MemNode::Memory));
 939           } else if (n->is_ArrayCopy()) {
 940             // Disconnect ArrayCopy node
 941             ArrayCopyNode* ac = n->as_ArrayCopy();
 942             assert(ac->is_clonebasic(), "unexpected array copy kind");
 943             Node* membar_after = ac->proj_out(TypeFunc::Control)->unique_ctrl_out();
 944             disconnect_projections(ac, _igvn);
 945             assert(alloc->in(0)->is_Proj() && alloc->in(0)->in(0)->Opcode() == Op_MemBarCPUOrder, "mem barrier expected before allocation");
 946             Node* membar_before = alloc->in(0)->in(0);
 947             disconnect_projections(membar_before->as_MemBar(), _igvn);
 948             if (membar_after->is_MemBar()) {
 949               disconnect_projections(membar_after->as_MemBar(), _igvn);
 950             }
 951           } else {
 952             eliminate_gc_barrier(n);
 953           }
 954           k -= (oc2 - use->outcnt());
 955         }
 956         if (UseShenandoahGC) {
 957           _igvn.remove_dead_node(use);
 958         }
 959       } else if (use->is_ArrayCopy()) {
 960         // Disconnect ArrayCopy node
 961         ArrayCopyNode* ac = use->as_ArrayCopy();
 962         assert(ac->is_arraycopy_validated() ||
 963                ac->is_copyof_validated() ||
 964                ac->is_copyofrange_validated(), "unsupported");
 965         CallProjections callprojs;
 966         ac->extract_projections(&callprojs, true);
 967 
 968         _igvn.replace_node(callprojs.fallthrough_ioproj, ac->in(TypeFunc::I_O));
 969         _igvn.replace_node(callprojs.fallthrough_memproj, ac->in(TypeFunc::Memory));
 970         _igvn.replace_node(callprojs.fallthrough_catchproj, ac->in(TypeFunc::Control));
 971 
 972         // Set control to top. IGVN will remove the remaining projections
 973         ac->set_req(0, top());
 974         ac->replace_edge(res, top());
 975 
 976         // Disconnect src right away: it can help find new
 977         // opportunities for allocation elimination


< prev index next >