209 case Op_LoadD:
210 case Op_LoadF:
211 case Op_LoadI:
212 case Op_LoadL:
213 case Op_LoadP:
214 case Op_LoadN:
215 case Op_LoadS:
216 case Op_LoadKlass:
217 case Op_LoadNKlass:
218 case Op_LoadRange:
219 case Op_LoadD_unaligned:
220 case Op_LoadL_unaligned:
221 assert(mach->in(2) == val, "should be address");
222 break;
223 case Op_StoreB:
224 case Op_StoreC:
225 case Op_StoreD:
226 case Op_StoreF:
227 case Op_StoreI:
228 case Op_StoreL:
229 case Op_StoreP:
230 case Op_StoreN:
231 case Op_StoreNKlass:
232 was_store = true; // Memory op is a store op
233 // Stores will have their address in slot 2 (memory in slot 1).
234 // If the value being nul-checked is in another slot, it means we
235 // are storing the checked value, which does NOT check the value!
236 if( mach->in(2) != val ) continue;
237 break; // Found a memory op?
238 case Op_StrComp:
239 case Op_StrEquals:
240 case Op_StrIndexOf:
241 case Op_StrIndexOfChar:
242 case Op_AryEq:
243 case Op_VectorizedHashCode:
244 case Op_StrInflatedCopy:
245 case Op_StrCompressedCopy:
246 case Op_EncodeISOArray:
247 case Op_CountPositives:
248 // Not a legit memory op for implicit null check regardless of
297 const TypePtr *adr_type = nullptr; // Do not need this return value here
298 const Node* base = mach->get_base_and_disp(offset, adr_type);
299 if (base == nullptr || base == NodeSentinel) {
300 // Narrow oop address doesn't have base, only index.
301 // Give up if offset is beyond page size or if heap base is not protected.
302 if (val->bottom_type()->isa_narrowoop() &&
303 (MacroAssembler::needs_explicit_null_check(offset) ||
304 !CompressedOops::use_implicit_null_checks()))
305 continue;
306 // cannot reason about it; is probably not implicit null exception
307 } else {
308 const TypePtr* tptr;
309 if ((UseCompressedOops && CompressedOops::shift() == 0) || CompressedKlassPointers::shift() == 0) {
310 // 32-bits narrow oop can be the base of address expressions
311 tptr = base->get_ptr_type();
312 } else {
313 // only regular oops are expected here
314 tptr = base->bottom_type()->is_ptr();
315 }
316 // Give up if offset is not a compile-time constant.
317 if (offset == Type::OffsetBot || tptr->_offset == Type::OffsetBot)
318 continue;
319 offset += tptr->_offset; // correct if base is offsetted
320 // Give up if reference is beyond page size.
321 if (MacroAssembler::needs_explicit_null_check(offset))
322 continue;
323 // Give up if base is a decode node and the heap base is not protected.
324 if (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_DecodeN &&
325 !CompressedOops::use_implicit_null_checks())
326 continue;
327 }
328 }
329
330 // Check ctrl input to see if the null-check dominates the memory op
331 Block *cb = get_block_for_node(mach);
332 cb = cb->_idom; // Always hoist at least 1 block
333 if( !was_store ) { // Stores can be hoisted only one block
334 while( cb->_dom_depth > (block->_dom_depth + 1))
335 cb = cb->_idom; // Hoist loads as far as we want
336 // The non-null-block should dominate the memory op, too. Live
337 // range spilling will insert a spill in the non-null-block if it is
338 // needs to spill the memory op for an implicit null check.
339 if (cb->_dom_depth == (block->_dom_depth + 1)) {
344 if( cb != block ) continue;
345
346 // Found a memory user; see if it can be hoisted to check-block
347 uint vidx = 0; // Capture index of value into memop
348 uint j;
349 for( j = mach->req()-1; j > 0; j-- ) {
350 if( mach->in(j) == val ) {
351 vidx = j;
352 // Ignore DecodeN val which could be hoisted to where needed.
353 if( is_decoden ) continue;
354 }
355 if (mach->in(j)->is_MachTemp()) {
356 assert(mach->in(j)->outcnt() == 1, "MachTemp nodes should not be shared");
357 // Ignore MachTemp inputs, they can be safely hoisted with the candidate.
358 // MachTemp nodes have no inputs themselves and are only used to reserve
359 // a scratch register for the implementation of the node (e.g. in
360 // late-expanded GC barriers).
361 continue;
362 }
363 // Block of memory-op input
364 Block *inb = get_block_for_node(mach->in(j));
365 Block *b = block; // Start from nul check
366 while( b != inb && b->_dom_depth > inb->_dom_depth )
367 b = b->_idom; // search upwards for input
368 // See if input dominates null check
369 if( b != inb )
370 break;
371 }
372 if( j > 0 )
373 continue;
374 Block *mb = get_block_for_node(mach);
375 // Hoisting stores requires more checks for the anti-dependence case.
376 // Give up hoisting if we have to move the store past any load.
377 if (was_store) {
378 // Make sure control does not do a merge (would have to check allpaths)
379 if (mb->num_preds() != 2) {
380 continue;
381 }
382 // mach is a store, hence block is the immediate dominator of mb.
383 // Due to the null-check shape of block (where its successors cannot re-join),
384 // block must be the direct predecessor of mb.
417 // ---- Found an implicit null check
418 #ifndef PRODUCT
419 extern uint implicit_null_checks;
420 implicit_null_checks++;
421 #endif
422
423 if( is_decoden ) {
424 // Check if we need to hoist decodeHeapOop_not_null first.
425 Block *valb = get_block_for_node(val);
426 if( block != valb && block->_dom_depth < valb->_dom_depth ) {
427 // Hoist it up to the end of the test block together with its inputs if they exist.
428 for (uint i = 2; i < val->req(); i++) {
429 // DecodeN has 2 regular inputs + optional MachTemp or load Base inputs.
430 // Inputs of val may already be early enough, but if not move them together with val.
431 ensure_node_is_at_block_or_above(val->in(i), block);
432 }
433 move_node_and_its_projections_to_block(val, block);
434 }
435 }
436
437 // Move any MachTemp inputs to the end of the test block.
438 for (uint i = 0; i < best->req(); i++) {
439 Node* n = best->in(i);
440 if (n == nullptr || !n->is_MachTemp()) {
441 continue;
442 }
443 ensure_node_is_at_block_or_above(n, block);
444 }
445
446 // Hoist the memory candidate up to the end of the test block.
447 move_node_and_its_projections_to_block(best, block);
448
449 // Move the control dependence if it is pinned to not-null block.
450 // Don't change it in other cases: null or dominating control.
451 Node* ctrl = best->in(0);
452 if (ctrl != nullptr && get_block_for_node(ctrl) == not_null_block) {
453 // Set it to control edge of null check.
454 best->set_req(0, proj->in(0)->in(0));
455 }
456
725 if (src == 0) continue;
726 LRG& lrg_src = _regalloc->lrgs(src);
727 // detect if the live range ends or not
728 if (liveout->member(src) == false) {
729 lrg_ends = true;
730 for (DUIterator_Fast jmax, j = src_n->fast_outs(jmax); j < jmax; j++) {
731 Node* m = src_n->fast_out(j); // Get user
732 if (m == n) continue;
733 if (!m->is_Mach()) continue;
734 MachNode *mach = m->as_Mach();
735 bool src_matches = false;
736 int iop = mach->ideal_Opcode();
737
738 switch (iop) {
739 case Op_StoreB:
740 case Op_StoreC:
741 case Op_StoreD:
742 case Op_StoreF:
743 case Op_StoreI:
744 case Op_StoreL:
745 case Op_StoreP:
746 case Op_StoreN:
747 case Op_StoreVector:
748 case Op_StoreVectorMasked:
749 case Op_StoreVectorScatter:
750 case Op_StoreVectorScatterMasked:
751 case Op_StoreNKlass:
752 for (uint k = 1; k < m->req(); k++) {
753 Node *in = m->in(k);
754 if (in == src_n) {
755 src_matches = true;
756 break;
757 }
758 }
759 break;
760
761 default:
762 src_matches = true;
763 break;
764 }
892 // Children of projections are now all ready
893 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
894 Node* m = n->fast_out(j); // Get user
895 if(get_block_for_node(m) != block) {
896 continue;
897 }
898 if( m->is_Phi() ) continue;
899 int m_cnt = ready_cnt.at(m->_idx) - 1;
900 ready_cnt.at_put(m->_idx, m_cnt);
901 if( m_cnt == 0 )
902 worklist.push(m);
903 }
904
905 }
906
907 // Act as if the call defines the Frame Pointer.
908 // Certainly the FP is alive and well after the call.
909 regs.insert(_matcher.c_frame_pointer());
910
911 // Set all registers killed and not already defined by the call.
912 uint r_cnt = mcall->tf()->range()->cnt();
913 int op = mcall->ideal_Opcode();
914 MachProjNode* proj = new MachProjNode(mcall, r_cnt + 1, RegMask::EMPTY, MachProjNode::fat_proj);
915 map_node_to_block(proj, block);
916 block->insert_node(proj, node_cnt++);
917
918 // Select the right register save policy.
919 const char *save_policy = nullptr;
920 switch (op) {
921 case Op_CallRuntime:
922 case Op_CallLeaf:
923 case Op_CallLeafNoFP:
924 case Op_CallLeafVector:
925 // Calling C code so use C calling convention
926 save_policy = _matcher._c_reg_save_policy;
927 break;
928
929 case Op_CallStaticJava:
930 case Op_CallDynamicJava:
931 // Calling Java code so use Java calling convention
932 save_policy = _matcher._register_save_policy;
|
209 case Op_LoadD:
210 case Op_LoadF:
211 case Op_LoadI:
212 case Op_LoadL:
213 case Op_LoadP:
214 case Op_LoadN:
215 case Op_LoadS:
216 case Op_LoadKlass:
217 case Op_LoadNKlass:
218 case Op_LoadRange:
219 case Op_LoadD_unaligned:
220 case Op_LoadL_unaligned:
221 assert(mach->in(2) == val, "should be address");
222 break;
223 case Op_StoreB:
224 case Op_StoreC:
225 case Op_StoreD:
226 case Op_StoreF:
227 case Op_StoreI:
228 case Op_StoreL:
229 case Op_StoreLSpecial:
230 case Op_StoreP:
231 case Op_StoreN:
232 case Op_StoreNKlass:
233 was_store = true; // Memory op is a store op
234 // Stores will have their address in slot 2 (memory in slot 1).
235 // If the value being nul-checked is in another slot, it means we
236 // are storing the checked value, which does NOT check the value!
237 if( mach->in(2) != val ) continue;
238 break; // Found a memory op?
239 case Op_StrComp:
240 case Op_StrEquals:
241 case Op_StrIndexOf:
242 case Op_StrIndexOfChar:
243 case Op_AryEq:
244 case Op_VectorizedHashCode:
245 case Op_StrInflatedCopy:
246 case Op_StrCompressedCopy:
247 case Op_EncodeISOArray:
248 case Op_CountPositives:
249 // Not a legit memory op for implicit null check regardless of
298 const TypePtr *adr_type = nullptr; // Do not need this return value here
299 const Node* base = mach->get_base_and_disp(offset, adr_type);
300 if (base == nullptr || base == NodeSentinel) {
301 // Narrow oop address doesn't have base, only index.
302 // Give up if offset is beyond page size or if heap base is not protected.
303 if (val->bottom_type()->isa_narrowoop() &&
304 (MacroAssembler::needs_explicit_null_check(offset) ||
305 !CompressedOops::use_implicit_null_checks()))
306 continue;
307 // cannot reason about it; is probably not implicit null exception
308 } else {
309 const TypePtr* tptr;
310 if ((UseCompressedOops && CompressedOops::shift() == 0) || CompressedKlassPointers::shift() == 0) {
311 // 32-bits narrow oop can be the base of address expressions
312 tptr = base->get_ptr_type();
313 } else {
314 // only regular oops are expected here
315 tptr = base->bottom_type()->is_ptr();
316 }
317 // Give up if offset is not a compile-time constant.
318 if (offset == Type::OffsetBot || tptr->offset() == Type::OffsetBot)
319 continue;
320 offset += tptr->offset(); // correct if base is offsetted
321 // Give up if reference is beyond page size.
322 if (MacroAssembler::needs_explicit_null_check(offset))
323 continue;
324 // Give up if base is a decode node and the heap base is not protected.
325 if (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_DecodeN &&
326 !CompressedOops::use_implicit_null_checks())
327 continue;
328 }
329 }
330
331 // Check ctrl input to see if the null-check dominates the memory op
332 Block *cb = get_block_for_node(mach);
333 cb = cb->_idom; // Always hoist at least 1 block
334 if( !was_store ) { // Stores can be hoisted only one block
335 while( cb->_dom_depth > (block->_dom_depth + 1))
336 cb = cb->_idom; // Hoist loads as far as we want
337 // The non-null-block should dominate the memory op, too. Live
338 // range spilling will insert a spill in the non-null-block if it is
339 // needs to spill the memory op for an implicit null check.
340 if (cb->_dom_depth == (block->_dom_depth + 1)) {
345 if( cb != block ) continue;
346
347 // Found a memory user; see if it can be hoisted to check-block
348 uint vidx = 0; // Capture index of value into memop
349 uint j;
350 for( j = mach->req()-1; j > 0; j-- ) {
351 if( mach->in(j) == val ) {
352 vidx = j;
353 // Ignore DecodeN val which could be hoisted to where needed.
354 if( is_decoden ) continue;
355 }
356 if (mach->in(j)->is_MachTemp()) {
357 assert(mach->in(j)->outcnt() == 1, "MachTemp nodes should not be shared");
358 // Ignore MachTemp inputs, they can be safely hoisted with the candidate.
359 // MachTemp nodes have no inputs themselves and are only used to reserve
360 // a scratch register for the implementation of the node (e.g. in
361 // late-expanded GC barriers).
362 continue;
363 }
364 // Block of memory-op input
365 Block* inb = get_block_for_node(mach->in(j));
366 if (mach->in(j)->is_Con() && mach->in(j)->req() == 1 && inb == get_block_for_node(mach)) {
367 // Ignore constant loads scheduled in the same block (we can simply hoist them as well)
368 continue;
369 }
370 Block *b = block; // Start from nul check
371 while( b != inb && b->_dom_depth > inb->_dom_depth )
372 b = b->_idom; // search upwards for input
373 // See if input dominates null check
374 if( b != inb )
375 break;
376 }
377 if( j > 0 )
378 continue;
379 Block *mb = get_block_for_node(mach);
380 // Hoisting stores requires more checks for the anti-dependence case.
381 // Give up hoisting if we have to move the store past any load.
382 if (was_store) {
383 // Make sure control does not do a merge (would have to check allpaths)
384 if (mb->num_preds() != 2) {
385 continue;
386 }
387 // mach is a store, hence block is the immediate dominator of mb.
388 // Due to the null-check shape of block (where its successors cannot re-join),
389 // block must be the direct predecessor of mb.
422 // ---- Found an implicit null check
423 #ifndef PRODUCT
424 extern uint implicit_null_checks;
425 implicit_null_checks++;
426 #endif
427
428 if( is_decoden ) {
429 // Check if we need to hoist decodeHeapOop_not_null first.
430 Block *valb = get_block_for_node(val);
431 if( block != valb && block->_dom_depth < valb->_dom_depth ) {
432 // Hoist it up to the end of the test block together with its inputs if they exist.
433 for (uint i = 2; i < val->req(); i++) {
434 // DecodeN has 2 regular inputs + optional MachTemp or load Base inputs.
435 // Inputs of val may already be early enough, but if not move them together with val.
436 ensure_node_is_at_block_or_above(val->in(i), block);
437 }
438 move_node_and_its_projections_to_block(val, block);
439 }
440 }
441
442 // Hoist constant load inputs as well.
443 for (uint i = 1; i < best->req(); ++i) {
444 Node* n = best->in(i);
445 if (n->is_Con() && get_block_for_node(n) == get_block_for_node(best)) {
446 get_block_for_node(n)->find_remove(n);
447 block->add_inst(n);
448 map_node_to_block(n, block);
449 // Constant loads may kill flags (for example, when XORing a register).
450 // Check for flag-killing projections that also need to be hoisted.
451 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
452 Node* proj = n->fast_out(j);
453 if (proj->is_MachProj()) {
454 get_block_for_node(proj)->find_remove(proj);
455 block->add_inst(proj);
456 map_node_to_block(proj, block);
457 }
458 }
459 }
460 }
461
462
463 // Move any MachTemp inputs to the end of the test block.
464 for (uint i = 0; i < best->req(); i++) {
465 Node* n = best->in(i);
466 if (n == nullptr || !n->is_MachTemp()) {
467 continue;
468 }
469 ensure_node_is_at_block_or_above(n, block);
470 }
471
472 // Hoist the memory candidate up to the end of the test block.
473 move_node_and_its_projections_to_block(best, block);
474
475 // Move the control dependence if it is pinned to not-null block.
476 // Don't change it in other cases: null or dominating control.
477 Node* ctrl = best->in(0);
478 if (ctrl != nullptr && get_block_for_node(ctrl) == not_null_block) {
479 // Set it to control edge of null check.
480 best->set_req(0, proj->in(0)->in(0));
481 }
482
751 if (src == 0) continue;
752 LRG& lrg_src = _regalloc->lrgs(src);
753 // detect if the live range ends or not
754 if (liveout->member(src) == false) {
755 lrg_ends = true;
756 for (DUIterator_Fast jmax, j = src_n->fast_outs(jmax); j < jmax; j++) {
757 Node* m = src_n->fast_out(j); // Get user
758 if (m == n) continue;
759 if (!m->is_Mach()) continue;
760 MachNode *mach = m->as_Mach();
761 bool src_matches = false;
762 int iop = mach->ideal_Opcode();
763
764 switch (iop) {
765 case Op_StoreB:
766 case Op_StoreC:
767 case Op_StoreD:
768 case Op_StoreF:
769 case Op_StoreI:
770 case Op_StoreL:
771 case Op_StoreLSpecial:
772 case Op_StoreP:
773 case Op_StoreN:
774 case Op_StoreVector:
775 case Op_StoreVectorMasked:
776 case Op_StoreVectorScatter:
777 case Op_StoreVectorScatterMasked:
778 case Op_StoreNKlass:
779 for (uint k = 1; k < m->req(); k++) {
780 Node *in = m->in(k);
781 if (in == src_n) {
782 src_matches = true;
783 break;
784 }
785 }
786 break;
787
788 default:
789 src_matches = true;
790 break;
791 }
919 // Children of projections are now all ready
920 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
921 Node* m = n->fast_out(j); // Get user
922 if(get_block_for_node(m) != block) {
923 continue;
924 }
925 if( m->is_Phi() ) continue;
926 int m_cnt = ready_cnt.at(m->_idx) - 1;
927 ready_cnt.at_put(m->_idx, m_cnt);
928 if( m_cnt == 0 )
929 worklist.push(m);
930 }
931
932 }
933
934 // Act as if the call defines the Frame Pointer.
935 // Certainly the FP is alive and well after the call.
936 regs.insert(_matcher.c_frame_pointer());
937
938 // Set all registers killed and not already defined by the call.
939 uint r_cnt = mcall->tf()->range_cc()->cnt();
940 int op = mcall->ideal_Opcode();
941 MachProjNode* proj = new MachProjNode(mcall, r_cnt + 1, RegMask::EMPTY, MachProjNode::fat_proj);
942 map_node_to_block(proj, block);
943 block->insert_node(proj, node_cnt++);
944
945 // Select the right register save policy.
946 const char *save_policy = nullptr;
947 switch (op) {
948 case Op_CallRuntime:
949 case Op_CallLeaf:
950 case Op_CallLeafNoFP:
951 case Op_CallLeafVector:
952 // Calling C code so use C calling convention
953 save_policy = _matcher._c_reg_save_policy;
954 break;
955
956 case Op_CallStaticJava:
957 case Op_CallDynamicJava:
958 // Calling Java code so use Java calling convention
959 save_policy = _matcher._register_save_policy;
|