< prev index next >

src/hotspot/share/opto/lcm.cpp

Print this page

 209     case Op_LoadD:
 210     case Op_LoadF:
 211     case Op_LoadI:
 212     case Op_LoadL:
 213     case Op_LoadP:
 214     case Op_LoadN:
 215     case Op_LoadS:
 216     case Op_LoadKlass:
 217     case Op_LoadNKlass:
 218     case Op_LoadRange:
 219     case Op_LoadD_unaligned:
 220     case Op_LoadL_unaligned:
 221       assert(mach->in(2) == val, "should be address");
 222       break;
 223     case Op_StoreB:
 224     case Op_StoreC:
 225     case Op_StoreD:
 226     case Op_StoreF:
 227     case Op_StoreI:
 228     case Op_StoreL:

 229     case Op_StoreP:
 230     case Op_StoreN:
 231     case Op_StoreNKlass:
 232       was_store = true;         // Memory op is a store op
 233       // Stores will have their address in slot 2 (memory in slot 1).
 234       // If the value being nul-checked is in another slot, it means we
 235       // are storing the checked value, which does NOT check the value!
 236       if( mach->in(2) != val ) continue;
 237       break;                    // Found a memory op?
 238     case Op_StrComp:
 239     case Op_StrEquals:
 240     case Op_StrIndexOf:
 241     case Op_StrIndexOfChar:
 242     case Op_AryEq:
 243     case Op_VectorizedHashCode:
 244     case Op_StrInflatedCopy:
 245     case Op_StrCompressedCopy:
 246     case Op_EncodeISOArray:
 247     case Op_CountPositives:
 248       // Not a legit memory op for implicit null check regardless of

 298       const Node* base = mach->get_base_and_disp(offset, adr_type);
 299       if (base == nullptr || base == NodeSentinel) {
 300         // Narrow oop address doesn't have base, only index.
 301         // Give up if offset is beyond page size or if heap base is not protected.
 302         if (val->bottom_type()->isa_narrowoop() &&
 303             (MacroAssembler::needs_explicit_null_check(offset) ||
 304              !CompressedOops::use_implicit_null_checks()))
 305           continue;
 306         // cannot reason about it; is probably not implicit null exception
 307       } else {
 308         const TypePtr* tptr;
 309         if ((UseCompressedOops && CompressedOops::shift() == 0) ||
 310             (UseCompressedClassPointers && CompressedKlassPointers::shift() == 0)) {
 311           // 32-bits narrow oop can be the base of address expressions
 312           tptr = base->get_ptr_type();
 313         } else {
 314           // only regular oops are expected here
 315           tptr = base->bottom_type()->is_ptr();
 316         }
 317         // Give up if offset is not a compile-time constant.
 318         if (offset == Type::OffsetBot || tptr->_offset == Type::OffsetBot)
 319           continue;
 320         offset += tptr->_offset; // correct if base is offsetted
 321         // Give up if reference is beyond page size.
 322         if (MacroAssembler::needs_explicit_null_check(offset))
 323           continue;
 324         // Give up if base is a decode node and the heap base is not protected.
 325         if (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_DecodeN &&
 326             !CompressedOops::use_implicit_null_checks())
 327           continue;
 328       }
 329     }
 330 
 331     // Check ctrl input to see if the null-check dominates the memory op
 332     Block *cb = get_block_for_node(mach);
 333     cb = cb->_idom;             // Always hoist at least 1 block
 334     if( !was_store ) {          // Stores can be hoisted only one block
 335       while( cb->_dom_depth > (block->_dom_depth + 1))
 336         cb = cb->_idom;         // Hoist loads as far as we want
 337       // The non-null-block should dominate the memory op, too. Live
 338       // range spilling will insert a spill in the non-null-block if it is
 339       // needs to spill the memory op for an implicit null check.
 340       if (cb->_dom_depth == (block->_dom_depth + 1)) {

 345     if( cb != block ) continue;
 346 
 347     // Found a memory user; see if it can be hoisted to check-block
 348     uint vidx = 0;              // Capture index of value into memop
 349     uint j;
 350     for( j = mach->req()-1; j > 0; j-- ) {
 351       if( mach->in(j) == val ) {
 352         vidx = j;
 353         // Ignore DecodeN val which could be hoisted to where needed.
 354         if( is_decoden ) continue;
 355       }
 356       if (mach->in(j)->is_MachTemp()) {
 357         assert(mach->in(j)->outcnt() == 1, "MachTemp nodes should not be shared");
 358         // Ignore MachTemp inputs, they can be safely hoisted with the candidate.
 359         // MachTemp nodes have no inputs themselves and are only used to reserve
 360         // a scratch register for the implementation of the node (e.g. in
 361         // late-expanded GC barriers).
 362         continue;
 363       }
 364       // Block of memory-op input
 365       Block *inb = get_block_for_node(mach->in(j));




 366       Block *b = block;          // Start from nul check
 367       while( b != inb && b->_dom_depth > inb->_dom_depth )
 368         b = b->_idom;           // search upwards for input
 369       // See if input dominates null check
 370       if( b != inb )
 371         break;
 372     }
 373     if( j > 0 )
 374       continue;
 375     Block *mb = get_block_for_node(mach);
 376     // Hoisting stores requires more checks for the anti-dependence case.
 377     // Give up hoisting if we have to move the store past any load.
 378     if (was_store) {
 379        // Make sure control does not do a merge (would have to check allpaths)
 380        if (mb->num_preds() != 2) {
 381          continue;
 382        }
 383        // mach is a store, hence block is the immediate dominator of mb.
 384        // Due to the null-check shape of block (where its successors cannot re-join),
 385        // block must be the direct predecessor of mb.

 418   // ---- Found an implicit null check
 419 #ifndef PRODUCT
 420   extern uint implicit_null_checks;
 421   implicit_null_checks++;
 422 #endif
 423 
 424   if( is_decoden ) {
 425     // Check if we need to hoist decodeHeapOop_not_null first.
 426     Block *valb = get_block_for_node(val);
 427     if( block != valb && block->_dom_depth < valb->_dom_depth ) {
 428       // Hoist it up to the end of the test block together with its inputs if they exist.
 429       for (uint i = 2; i < val->req(); i++) {
 430         // DecodeN has 2 regular inputs + optional MachTemp or load Base inputs.
 431         // Inputs of val may already be early enough, but if not move them together with val.
 432         ensure_node_is_at_block_or_above(val->in(i), block);
 433       }
 434       move_node_and_its_projections_to_block(val, block);
 435     }
 436   }
 437 





















 438   // Move any MachTemp inputs to the end of the test block.
 439   for (uint i = 0; i < best->req(); i++) {
 440     Node* n = best->in(i);
 441     if (n == nullptr || !n->is_MachTemp()) {
 442       continue;
 443     }
 444     ensure_node_is_at_block_or_above(n, block);
 445   }
 446 
 447   // Hoist the memory candidate up to the end of the test block.
 448   move_node_and_its_projections_to_block(best, block);
 449 
 450   // Move the control dependence if it is pinned to not-null block.
 451   // Don't change it in other cases: null or dominating control.
 452   Node* ctrl = best->in(0);
 453   if (ctrl != nullptr && get_block_for_node(ctrl) == not_null_block) {
 454     // Set it to control edge of null check.
 455     best->set_req(0, proj->in(0)->in(0));
 456   }
 457 

 726     if (src == 0) continue;
 727     LRG& lrg_src = _regalloc->lrgs(src);
 728     // detect if the live range ends or not
 729     if (liveout->member(src) == false) {
 730       lrg_ends = true;
 731       for (DUIterator_Fast jmax, j = src_n->fast_outs(jmax); j < jmax; j++) {
 732         Node* m = src_n->fast_out(j); // Get user
 733         if (m == n) continue;
 734         if (!m->is_Mach()) continue;
 735         MachNode *mach = m->as_Mach();
 736         bool src_matches = false;
 737         int iop = mach->ideal_Opcode();
 738 
 739         switch (iop) {
 740         case Op_StoreB:
 741         case Op_StoreC:
 742         case Op_StoreD:
 743         case Op_StoreF:
 744         case Op_StoreI:
 745         case Op_StoreL:

 746         case Op_StoreP:
 747         case Op_StoreN:
 748         case Op_StoreVector:
 749         case Op_StoreVectorMasked:
 750         case Op_StoreVectorScatter:
 751         case Op_StoreVectorScatterMasked:
 752         case Op_StoreNKlass:
 753           for (uint k = 1; k < m->req(); k++) {
 754             Node *in = m->in(k);
 755             if (in == src_n) {
 756               src_matches = true;
 757               break;
 758             }
 759           }
 760           break;
 761 
 762         default:
 763           src_matches = true;
 764           break;
 765         }

 892     // Children of projections are now all ready
 893     for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
 894       Node* m = n->fast_out(j); // Get user
 895       if(get_block_for_node(m) != block) {
 896         continue;
 897       }
 898       if( m->is_Phi() ) continue;
 899       int m_cnt = ready_cnt.at(m->_idx) - 1;
 900       ready_cnt.at_put(m->_idx, m_cnt);
 901       if( m_cnt == 0 )
 902         worklist.push(m);
 903     }
 904 
 905   }
 906 
 907   // Act as if the call defines the Frame Pointer.
 908   // Certainly the FP is alive and well after the call.
 909   regs.Insert(_matcher.c_frame_pointer());
 910 
 911   // Set all registers killed and not already defined by the call.
 912   uint r_cnt = mcall->tf()->range()->cnt();
 913   int op = mcall->ideal_Opcode();
 914   MachProjNode *proj = new MachProjNode( mcall, r_cnt+1, RegMask::Empty, MachProjNode::fat_proj );
 915   map_node_to_block(proj, block);
 916   block->insert_node(proj, node_cnt++);
 917 
 918   // Select the right register save policy.
 919   const char *save_policy = nullptr;
 920   switch (op) {
 921     case Op_CallRuntime:
 922     case Op_CallLeaf:
 923     case Op_CallLeafNoFP:
 924     case Op_CallLeafVector:
 925       // Calling C code so use C calling convention
 926       save_policy = _matcher._c_reg_save_policy;
 927       break;
 928 
 929     case Op_CallStaticJava:
 930     case Op_CallDynamicJava:
 931       // Calling Java code so use Java calling convention
 932       save_policy = _matcher._register_save_policy;

 209     case Op_LoadD:
 210     case Op_LoadF:
 211     case Op_LoadI:
 212     case Op_LoadL:
 213     case Op_LoadP:
 214     case Op_LoadN:
 215     case Op_LoadS:
 216     case Op_LoadKlass:
 217     case Op_LoadNKlass:
 218     case Op_LoadRange:
 219     case Op_LoadD_unaligned:
 220     case Op_LoadL_unaligned:
 221       assert(mach->in(2) == val, "should be address");
 222       break;
 223     case Op_StoreB:
 224     case Op_StoreC:
 225     case Op_StoreD:
 226     case Op_StoreF:
 227     case Op_StoreI:
 228     case Op_StoreL:
 229     case Op_StoreLSpecial:
 230     case Op_StoreP:
 231     case Op_StoreN:
 232     case Op_StoreNKlass:
 233       was_store = true;         // Memory op is a store op
 234       // Stores will have their address in slot 2 (memory in slot 1).
 235       // If the value being nul-checked is in another slot, it means we
 236       // are storing the checked value, which does NOT check the value!
 237       if( mach->in(2) != val ) continue;
 238       break;                    // Found a memory op?
 239     case Op_StrComp:
 240     case Op_StrEquals:
 241     case Op_StrIndexOf:
 242     case Op_StrIndexOfChar:
 243     case Op_AryEq:
 244     case Op_VectorizedHashCode:
 245     case Op_StrInflatedCopy:
 246     case Op_StrCompressedCopy:
 247     case Op_EncodeISOArray:
 248     case Op_CountPositives:
 249       // Not a legit memory op for implicit null check regardless of

 299       const Node* base = mach->get_base_and_disp(offset, adr_type);
 300       if (base == nullptr || base == NodeSentinel) {
 301         // Narrow oop address doesn't have base, only index.
 302         // Give up if offset is beyond page size or if heap base is not protected.
 303         if (val->bottom_type()->isa_narrowoop() &&
 304             (MacroAssembler::needs_explicit_null_check(offset) ||
 305              !CompressedOops::use_implicit_null_checks()))
 306           continue;
 307         // cannot reason about it; is probably not implicit null exception
 308       } else {
 309         const TypePtr* tptr;
 310         if ((UseCompressedOops && CompressedOops::shift() == 0) ||
 311             (UseCompressedClassPointers && CompressedKlassPointers::shift() == 0)) {
 312           // 32-bits narrow oop can be the base of address expressions
 313           tptr = base->get_ptr_type();
 314         } else {
 315           // only regular oops are expected here
 316           tptr = base->bottom_type()->is_ptr();
 317         }
 318         // Give up if offset is not a compile-time constant.
 319         if (offset == Type::OffsetBot || tptr->offset() == Type::OffsetBot)
 320           continue;
 321         offset += tptr->offset(); // correct if base is offsetted
 322         // Give up if reference is beyond page size.
 323         if (MacroAssembler::needs_explicit_null_check(offset))
 324           continue;
 325         // Give up if base is a decode node and the heap base is not protected.
 326         if (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_DecodeN &&
 327             !CompressedOops::use_implicit_null_checks())
 328           continue;
 329       }
 330     }
 331 
 332     // Check ctrl input to see if the null-check dominates the memory op
 333     Block *cb = get_block_for_node(mach);
 334     cb = cb->_idom;             // Always hoist at least 1 block
 335     if( !was_store ) {          // Stores can be hoisted only one block
 336       while( cb->_dom_depth > (block->_dom_depth + 1))
 337         cb = cb->_idom;         // Hoist loads as far as we want
 338       // The non-null-block should dominate the memory op, too. Live
 339       // range spilling will insert a spill in the non-null-block if it is
 340       // needs to spill the memory op for an implicit null check.
 341       if (cb->_dom_depth == (block->_dom_depth + 1)) {

 346     if( cb != block ) continue;
 347 
 348     // Found a memory user; see if it can be hoisted to check-block
 349     uint vidx = 0;              // Capture index of value into memop
 350     uint j;
 351     for( j = mach->req()-1; j > 0; j-- ) {
 352       if( mach->in(j) == val ) {
 353         vidx = j;
 354         // Ignore DecodeN val which could be hoisted to where needed.
 355         if( is_decoden ) continue;
 356       }
 357       if (mach->in(j)->is_MachTemp()) {
 358         assert(mach->in(j)->outcnt() == 1, "MachTemp nodes should not be shared");
 359         // Ignore MachTemp inputs, they can be safely hoisted with the candidate.
 360         // MachTemp nodes have no inputs themselves and are only used to reserve
 361         // a scratch register for the implementation of the node (e.g. in
 362         // late-expanded GC barriers).
 363         continue;
 364       }
 365       // Block of memory-op input
 366       Block* inb = get_block_for_node(mach->in(j));
 367       if (mach->in(j)->is_Con() && mach->in(j)->req() == 1 && inb == get_block_for_node(mach)) {
 368         // Ignore constant loads scheduled in the same block (we can simply hoist them as well)
 369         continue;
 370       }
 371       Block *b = block;          // Start from nul check
 372       while( b != inb && b->_dom_depth > inb->_dom_depth )
 373         b = b->_idom;           // search upwards for input
 374       // See if input dominates null check
 375       if( b != inb )
 376         break;
 377     }
 378     if( j > 0 )
 379       continue;
 380     Block *mb = get_block_for_node(mach);
 381     // Hoisting stores requires more checks for the anti-dependence case.
 382     // Give up hoisting if we have to move the store past any load.
 383     if (was_store) {
 384        // Make sure control does not do a merge (would have to check allpaths)
 385        if (mb->num_preds() != 2) {
 386          continue;
 387        }
 388        // mach is a store, hence block is the immediate dominator of mb.
 389        // Due to the null-check shape of block (where its successors cannot re-join),
 390        // block must be the direct predecessor of mb.

 423   // ---- Found an implicit null check
 424 #ifndef PRODUCT
 425   extern uint implicit_null_checks;
 426   implicit_null_checks++;
 427 #endif
 428 
 429   if( is_decoden ) {
 430     // Check if we need to hoist decodeHeapOop_not_null first.
 431     Block *valb = get_block_for_node(val);
 432     if( block != valb && block->_dom_depth < valb->_dom_depth ) {
 433       // Hoist it up to the end of the test block together with its inputs if they exist.
 434       for (uint i = 2; i < val->req(); i++) {
 435         // DecodeN has 2 regular inputs + optional MachTemp or load Base inputs.
 436         // Inputs of val may already be early enough, but if not move them together with val.
 437         ensure_node_is_at_block_or_above(val->in(i), block);
 438       }
 439       move_node_and_its_projections_to_block(val, block);
 440     }
 441   }
 442 
 443   // Hoist constant load inputs as well.
 444   for (uint i = 1; i < best->req(); ++i) {
 445     Node* n = best->in(i);
 446     if (n->is_Con() && get_block_for_node(n) == get_block_for_node(best)) {
 447       get_block_for_node(n)->find_remove(n);
 448       block->add_inst(n);
 449       map_node_to_block(n, block);
 450       // Constant loads may kill flags (for example, when XORing a register).
 451       // Check for flag-killing projections that also need to be hoisted.
 452       for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
 453         Node* proj = n->fast_out(j);
 454         if (proj->is_MachProj()) {
 455           get_block_for_node(proj)->find_remove(proj);
 456           block->add_inst(proj);
 457           map_node_to_block(proj, block);
 458         }
 459       }
 460     }
 461   }
 462 
 463 
 464   // Move any MachTemp inputs to the end of the test block.
 465   for (uint i = 0; i < best->req(); i++) {
 466     Node* n = best->in(i);
 467     if (n == nullptr || !n->is_MachTemp()) {
 468       continue;
 469     }
 470     ensure_node_is_at_block_or_above(n, block);
 471   }
 472 
 473   // Hoist the memory candidate up to the end of the test block.
 474   move_node_and_its_projections_to_block(best, block);
 475 
 476   // Move the control dependence if it is pinned to not-null block.
 477   // Don't change it in other cases: null or dominating control.
 478   Node* ctrl = best->in(0);
 479   if (ctrl != nullptr && get_block_for_node(ctrl) == not_null_block) {
 480     // Set it to control edge of null check.
 481     best->set_req(0, proj->in(0)->in(0));
 482   }
 483 

 752     if (src == 0) continue;
 753     LRG& lrg_src = _regalloc->lrgs(src);
 754     // detect if the live range ends or not
 755     if (liveout->member(src) == false) {
 756       lrg_ends = true;
 757       for (DUIterator_Fast jmax, j = src_n->fast_outs(jmax); j < jmax; j++) {
 758         Node* m = src_n->fast_out(j); // Get user
 759         if (m == n) continue;
 760         if (!m->is_Mach()) continue;
 761         MachNode *mach = m->as_Mach();
 762         bool src_matches = false;
 763         int iop = mach->ideal_Opcode();
 764 
 765         switch (iop) {
 766         case Op_StoreB:
 767         case Op_StoreC:
 768         case Op_StoreD:
 769         case Op_StoreF:
 770         case Op_StoreI:
 771         case Op_StoreL:
 772         case Op_StoreLSpecial:
 773         case Op_StoreP:
 774         case Op_StoreN:
 775         case Op_StoreVector:
 776         case Op_StoreVectorMasked:
 777         case Op_StoreVectorScatter:
 778         case Op_StoreVectorScatterMasked:
 779         case Op_StoreNKlass:
 780           for (uint k = 1; k < m->req(); k++) {
 781             Node *in = m->in(k);
 782             if (in == src_n) {
 783               src_matches = true;
 784               break;
 785             }
 786           }
 787           break;
 788 
 789         default:
 790           src_matches = true;
 791           break;
 792         }

 919     // Children of projections are now all ready
 920     for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
 921       Node* m = n->fast_out(j); // Get user
 922       if(get_block_for_node(m) != block) {
 923         continue;
 924       }
 925       if( m->is_Phi() ) continue;
 926       int m_cnt = ready_cnt.at(m->_idx) - 1;
 927       ready_cnt.at_put(m->_idx, m_cnt);
 928       if( m_cnt == 0 )
 929         worklist.push(m);
 930     }
 931 
 932   }
 933 
 934   // Act as if the call defines the Frame Pointer.
 935   // Certainly the FP is alive and well after the call.
 936   regs.Insert(_matcher.c_frame_pointer());
 937 
 938   // Set all registers killed and not already defined by the call.
 939   uint r_cnt = mcall->tf()->range_cc()->cnt();
 940   int op = mcall->ideal_Opcode();
 941   MachProjNode *proj = new MachProjNode( mcall, r_cnt+1, RegMask::Empty, MachProjNode::fat_proj );
 942   map_node_to_block(proj, block);
 943   block->insert_node(proj, node_cnt++);
 944 
 945   // Select the right register save policy.
 946   const char *save_policy = nullptr;
 947   switch (op) {
 948     case Op_CallRuntime:
 949     case Op_CallLeaf:
 950     case Op_CallLeafNoFP:
 951     case Op_CallLeafVector:
 952       // Calling C code so use C calling convention
 953       save_policy = _matcher._c_reg_save_policy;
 954       break;
 955 
 956     case Op_CallStaticJava:
 957     case Op_CallDynamicJava:
 958       // Calling Java code so use Java calling convention
 959       save_policy = _matcher._register_save_policy;
< prev index next >