209 case Op_LoadD:
210 case Op_LoadF:
211 case Op_LoadI:
212 case Op_LoadL:
213 case Op_LoadP:
214 case Op_LoadN:
215 case Op_LoadS:
216 case Op_LoadKlass:
217 case Op_LoadNKlass:
218 case Op_LoadRange:
219 case Op_LoadD_unaligned:
220 case Op_LoadL_unaligned:
221 assert(mach->in(2) == val, "should be address");
222 break;
223 case Op_StoreB:
224 case Op_StoreC:
225 case Op_StoreD:
226 case Op_StoreF:
227 case Op_StoreI:
228 case Op_StoreL:
229 case Op_StoreP:
230 case Op_StoreN:
231 case Op_StoreNKlass:
232 was_store = true; // Memory op is a store op
233 // Stores will have their address in slot 2 (memory in slot 1).
234 // If the value being nul-checked is in another slot, it means we
235 // are storing the checked value, which does NOT check the value!
236 if( mach->in(2) != val ) continue;
237 break; // Found a memory op?
238 case Op_StrComp:
239 case Op_StrEquals:
240 case Op_StrIndexOf:
241 case Op_StrIndexOfChar:
242 case Op_AryEq:
243 case Op_VectorizedHashCode:
244 case Op_StrInflatedCopy:
245 case Op_StrCompressedCopy:
246 case Op_EncodeISOArray:
247 case Op_CountPositives:
248 // Not a legit memory op for implicit null check regardless of
298 const Node* base = mach->get_base_and_disp(offset, adr_type);
299 if (base == nullptr || base == NodeSentinel) {
300 // Narrow oop address doesn't have base, only index.
301 // Give up if offset is beyond page size or if heap base is not protected.
302 if (val->bottom_type()->isa_narrowoop() &&
303 (MacroAssembler::needs_explicit_null_check(offset) ||
304 !CompressedOops::use_implicit_null_checks()))
305 continue;
306 // cannot reason about it; is probably not implicit null exception
307 } else {
308 const TypePtr* tptr;
309 if ((UseCompressedOops && CompressedOops::shift() == 0) ||
310 (UseCompressedClassPointers && CompressedKlassPointers::shift() == 0)) {
311 // 32-bits narrow oop can be the base of address expressions
312 tptr = base->get_ptr_type();
313 } else {
314 // only regular oops are expected here
315 tptr = base->bottom_type()->is_ptr();
316 }
317 // Give up if offset is not a compile-time constant.
318 if (offset == Type::OffsetBot || tptr->_offset == Type::OffsetBot)
319 continue;
320 offset += tptr->_offset; // correct if base is offsetted
321 // Give up if reference is beyond page size.
322 if (MacroAssembler::needs_explicit_null_check(offset))
323 continue;
324 // Give up if base is a decode node and the heap base is not protected.
325 if (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_DecodeN &&
326 !CompressedOops::use_implicit_null_checks())
327 continue;
328 }
329 }
330
331 // Check ctrl input to see if the null-check dominates the memory op
332 Block *cb = get_block_for_node(mach);
333 cb = cb->_idom; // Always hoist at least 1 block
334 if( !was_store ) { // Stores can be hoisted only one block
335 while( cb->_dom_depth > (block->_dom_depth + 1))
336 cb = cb->_idom; // Hoist loads as far as we want
337 // The non-null-block should dominate the memory op, too. Live
338 // range spilling will insert a spill in the non-null-block if it is
339 // needs to spill the memory op for an implicit null check.
340 if (cb->_dom_depth == (block->_dom_depth + 1)) {
345 if( cb != block ) continue;
346
347 // Found a memory user; see if it can be hoisted to check-block
348 uint vidx = 0; // Capture index of value into memop
349 uint j;
350 for( j = mach->req()-1; j > 0; j-- ) {
351 if( mach->in(j) == val ) {
352 vidx = j;
353 // Ignore DecodeN val which could be hoisted to where needed.
354 if( is_decoden ) continue;
355 }
356 if (mach->in(j)->is_MachTemp()) {
357 assert(mach->in(j)->outcnt() == 1, "MachTemp nodes should not be shared");
358 // Ignore MachTemp inputs, they can be safely hoisted with the candidate.
359 // MachTemp nodes have no inputs themselves and are only used to reserve
360 // a scratch register for the implementation of the node (e.g. in
361 // late-expanded GC barriers).
362 continue;
363 }
364 // Block of memory-op input
365 Block *inb = get_block_for_node(mach->in(j));
366 Block *b = block; // Start from nul check
367 while( b != inb && b->_dom_depth > inb->_dom_depth )
368 b = b->_idom; // search upwards for input
369 // See if input dominates null check
370 if( b != inb )
371 break;
372 }
373 if( j > 0 )
374 continue;
375 Block *mb = get_block_for_node(mach);
376 // Hoisting stores requires more checks for the anti-dependence case.
377 // Give up hoisting if we have to move the store past any load.
378 if (was_store) {
379 // Make sure control does not do a merge (would have to check allpaths)
380 if (mb->num_preds() != 2) {
381 continue;
382 }
383 // mach is a store, hence block is the immediate dominator of mb.
384 // Due to the null-check shape of block (where its successors cannot re-join),
385 // block must be the direct predecessor of mb.
418 // ---- Found an implicit null check
419 #ifndef PRODUCT
420 extern uint implicit_null_checks;
421 implicit_null_checks++;
422 #endif
423
424 if( is_decoden ) {
425 // Check if we need to hoist decodeHeapOop_not_null first.
426 Block *valb = get_block_for_node(val);
427 if( block != valb && block->_dom_depth < valb->_dom_depth ) {
428 // Hoist it up to the end of the test block together with its inputs if they exist.
429 for (uint i = 2; i < val->req(); i++) {
430 // DecodeN has 2 regular inputs + optional MachTemp or load Base inputs.
431 // Inputs of val may already be early enough, but if not move them together with val.
432 ensure_node_is_at_block_or_above(val->in(i), block);
433 }
434 move_node_and_its_projections_to_block(val, block);
435 }
436 }
437
438 // Move any MachTemp inputs to the end of the test block.
439 for (uint i = 0; i < best->req(); i++) {
440 Node* n = best->in(i);
441 if (n == nullptr || !n->is_MachTemp()) {
442 continue;
443 }
444 ensure_node_is_at_block_or_above(n, block);
445 }
446
447 // Hoist the memory candidate up to the end of the test block.
448 move_node_and_its_projections_to_block(best, block);
449
450 // Move the control dependence if it is pinned to not-null block.
451 // Don't change it in other cases: null or dominating control.
452 Node* ctrl = best->in(0);
453 if (ctrl != nullptr && get_block_for_node(ctrl) == not_null_block) {
454 // Set it to control edge of null check.
455 best->set_req(0, proj->in(0)->in(0));
456 }
457
726 if (src == 0) continue;
727 LRG& lrg_src = _regalloc->lrgs(src);
728 // detect if the live range ends or not
729 if (liveout->member(src) == false) {
730 lrg_ends = true;
731 for (DUIterator_Fast jmax, j = src_n->fast_outs(jmax); j < jmax; j++) {
732 Node* m = src_n->fast_out(j); // Get user
733 if (m == n) continue;
734 if (!m->is_Mach()) continue;
735 MachNode *mach = m->as_Mach();
736 bool src_matches = false;
737 int iop = mach->ideal_Opcode();
738
739 switch (iop) {
740 case Op_StoreB:
741 case Op_StoreC:
742 case Op_StoreD:
743 case Op_StoreF:
744 case Op_StoreI:
745 case Op_StoreL:
746 case Op_StoreP:
747 case Op_StoreN:
748 case Op_StoreVector:
749 case Op_StoreVectorMasked:
750 case Op_StoreVectorScatter:
751 case Op_StoreVectorScatterMasked:
752 case Op_StoreNKlass:
753 for (uint k = 1; k < m->req(); k++) {
754 Node *in = m->in(k);
755 if (in == src_n) {
756 src_matches = true;
757 break;
758 }
759 }
760 break;
761
762 default:
763 src_matches = true;
764 break;
765 }
892 // Children of projections are now all ready
893 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
894 Node* m = n->fast_out(j); // Get user
895 if(get_block_for_node(m) != block) {
896 continue;
897 }
898 if( m->is_Phi() ) continue;
899 int m_cnt = ready_cnt.at(m->_idx) - 1;
900 ready_cnt.at_put(m->_idx, m_cnt);
901 if( m_cnt == 0 )
902 worklist.push(m);
903 }
904
905 }
906
907 // Act as if the call defines the Frame Pointer.
908 // Certainly the FP is alive and well after the call.
909 regs.Insert(_matcher.c_frame_pointer());
910
911 // Set all registers killed and not already defined by the call.
912 uint r_cnt = mcall->tf()->range()->cnt();
913 int op = mcall->ideal_Opcode();
914 MachProjNode *proj = new MachProjNode( mcall, r_cnt+1, RegMask::Empty, MachProjNode::fat_proj );
915 map_node_to_block(proj, block);
916 block->insert_node(proj, node_cnt++);
917
918 // Select the right register save policy.
919 const char *save_policy = nullptr;
920 switch (op) {
921 case Op_CallRuntime:
922 case Op_CallLeaf:
923 case Op_CallLeafNoFP:
924 case Op_CallLeafVector:
925 // Calling C code so use C calling convention
926 save_policy = _matcher._c_reg_save_policy;
927 break;
928
929 case Op_CallStaticJava:
930 case Op_CallDynamicJava:
931 // Calling Java code so use Java calling convention
932 save_policy = _matcher._register_save_policy;
|
209 case Op_LoadD:
210 case Op_LoadF:
211 case Op_LoadI:
212 case Op_LoadL:
213 case Op_LoadP:
214 case Op_LoadN:
215 case Op_LoadS:
216 case Op_LoadKlass:
217 case Op_LoadNKlass:
218 case Op_LoadRange:
219 case Op_LoadD_unaligned:
220 case Op_LoadL_unaligned:
221 assert(mach->in(2) == val, "should be address");
222 break;
223 case Op_StoreB:
224 case Op_StoreC:
225 case Op_StoreD:
226 case Op_StoreF:
227 case Op_StoreI:
228 case Op_StoreL:
229 case Op_StoreLSpecial:
230 case Op_StoreP:
231 case Op_StoreN:
232 case Op_StoreNKlass:
233 was_store = true; // Memory op is a store op
234 // Stores will have their address in slot 2 (memory in slot 1).
235 // If the value being nul-checked is in another slot, it means we
236 // are storing the checked value, which does NOT check the value!
237 if( mach->in(2) != val ) continue;
238 break; // Found a memory op?
239 case Op_StrComp:
240 case Op_StrEquals:
241 case Op_StrIndexOf:
242 case Op_StrIndexOfChar:
243 case Op_AryEq:
244 case Op_VectorizedHashCode:
245 case Op_StrInflatedCopy:
246 case Op_StrCompressedCopy:
247 case Op_EncodeISOArray:
248 case Op_CountPositives:
249 // Not a legit memory op for implicit null check regardless of
299 const Node* base = mach->get_base_and_disp(offset, adr_type);
300 if (base == nullptr || base == NodeSentinel) {
301 // Narrow oop address doesn't have base, only index.
302 // Give up if offset is beyond page size or if heap base is not protected.
303 if (val->bottom_type()->isa_narrowoop() &&
304 (MacroAssembler::needs_explicit_null_check(offset) ||
305 !CompressedOops::use_implicit_null_checks()))
306 continue;
307 // cannot reason about it; is probably not implicit null exception
308 } else {
309 const TypePtr* tptr;
310 if ((UseCompressedOops && CompressedOops::shift() == 0) ||
311 (UseCompressedClassPointers && CompressedKlassPointers::shift() == 0)) {
312 // 32-bits narrow oop can be the base of address expressions
313 tptr = base->get_ptr_type();
314 } else {
315 // only regular oops are expected here
316 tptr = base->bottom_type()->is_ptr();
317 }
318 // Give up if offset is not a compile-time constant.
319 if (offset == Type::OffsetBot || tptr->offset() == Type::OffsetBot)
320 continue;
321 offset += tptr->offset(); // correct if base is offsetted
322 // Give up if reference is beyond page size.
323 if (MacroAssembler::needs_explicit_null_check(offset))
324 continue;
325 // Give up if base is a decode node and the heap base is not protected.
326 if (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_DecodeN &&
327 !CompressedOops::use_implicit_null_checks())
328 continue;
329 }
330 }
331
332 // Check ctrl input to see if the null-check dominates the memory op
333 Block *cb = get_block_for_node(mach);
334 cb = cb->_idom; // Always hoist at least 1 block
335 if( !was_store ) { // Stores can be hoisted only one block
336 while( cb->_dom_depth > (block->_dom_depth + 1))
337 cb = cb->_idom; // Hoist loads as far as we want
338 // The non-null-block should dominate the memory op, too. Live
339 // range spilling will insert a spill in the non-null-block if it is
340 // needs to spill the memory op for an implicit null check.
341 if (cb->_dom_depth == (block->_dom_depth + 1)) {
346 if( cb != block ) continue;
347
348 // Found a memory user; see if it can be hoisted to check-block
349 uint vidx = 0; // Capture index of value into memop
350 uint j;
351 for( j = mach->req()-1; j > 0; j-- ) {
352 if( mach->in(j) == val ) {
353 vidx = j;
354 // Ignore DecodeN val which could be hoisted to where needed.
355 if( is_decoden ) continue;
356 }
357 if (mach->in(j)->is_MachTemp()) {
358 assert(mach->in(j)->outcnt() == 1, "MachTemp nodes should not be shared");
359 // Ignore MachTemp inputs, they can be safely hoisted with the candidate.
360 // MachTemp nodes have no inputs themselves and are only used to reserve
361 // a scratch register for the implementation of the node (e.g. in
362 // late-expanded GC barriers).
363 continue;
364 }
365 // Block of memory-op input
366 Block* inb = get_block_for_node(mach->in(j));
367 if (mach->in(j)->is_Con() && mach->in(j)->req() == 1 && inb == get_block_for_node(mach)) {
368 // Ignore constant loads scheduled in the same block (we can simply hoist them as well)
369 continue;
370 }
371 Block *b = block; // Start from nul check
372 while( b != inb && b->_dom_depth > inb->_dom_depth )
373 b = b->_idom; // search upwards for input
374 // See if input dominates null check
375 if( b != inb )
376 break;
377 }
378 if( j > 0 )
379 continue;
380 Block *mb = get_block_for_node(mach);
381 // Hoisting stores requires more checks for the anti-dependence case.
382 // Give up hoisting if we have to move the store past any load.
383 if (was_store) {
384 // Make sure control does not do a merge (would have to check allpaths)
385 if (mb->num_preds() != 2) {
386 continue;
387 }
388 // mach is a store, hence block is the immediate dominator of mb.
389 // Due to the null-check shape of block (where its successors cannot re-join),
390 // block must be the direct predecessor of mb.
423 // ---- Found an implicit null check
424 #ifndef PRODUCT
425 extern uint implicit_null_checks;
426 implicit_null_checks++;
427 #endif
428
429 if( is_decoden ) {
430 // Check if we need to hoist decodeHeapOop_not_null first.
431 Block *valb = get_block_for_node(val);
432 if( block != valb && block->_dom_depth < valb->_dom_depth ) {
433 // Hoist it up to the end of the test block together with its inputs if they exist.
434 for (uint i = 2; i < val->req(); i++) {
435 // DecodeN has 2 regular inputs + optional MachTemp or load Base inputs.
436 // Inputs of val may already be early enough, but if not move them together with val.
437 ensure_node_is_at_block_or_above(val->in(i), block);
438 }
439 move_node_and_its_projections_to_block(val, block);
440 }
441 }
442
443 // Hoist constant load inputs as well.
444 for (uint i = 1; i < best->req(); ++i) {
445 Node* n = best->in(i);
446 if (n->is_Con() && get_block_for_node(n) == get_block_for_node(best)) {
447 get_block_for_node(n)->find_remove(n);
448 block->add_inst(n);
449 map_node_to_block(n, block);
450 // Constant loads may kill flags (for example, when XORing a register).
451 // Check for flag-killing projections that also need to be hoisted.
452 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
453 Node* proj = n->fast_out(j);
454 if (proj->is_MachProj()) {
455 get_block_for_node(proj)->find_remove(proj);
456 block->add_inst(proj);
457 map_node_to_block(proj, block);
458 }
459 }
460 }
461 }
462
463
464 // Move any MachTemp inputs to the end of the test block.
465 for (uint i = 0; i < best->req(); i++) {
466 Node* n = best->in(i);
467 if (n == nullptr || !n->is_MachTemp()) {
468 continue;
469 }
470 ensure_node_is_at_block_or_above(n, block);
471 }
472
473 // Hoist the memory candidate up to the end of the test block.
474 move_node_and_its_projections_to_block(best, block);
475
476 // Move the control dependence if it is pinned to not-null block.
477 // Don't change it in other cases: null or dominating control.
478 Node* ctrl = best->in(0);
479 if (ctrl != nullptr && get_block_for_node(ctrl) == not_null_block) {
480 // Set it to control edge of null check.
481 best->set_req(0, proj->in(0)->in(0));
482 }
483
752 if (src == 0) continue;
753 LRG& lrg_src = _regalloc->lrgs(src);
754 // detect if the live range ends or not
755 if (liveout->member(src) == false) {
756 lrg_ends = true;
757 for (DUIterator_Fast jmax, j = src_n->fast_outs(jmax); j < jmax; j++) {
758 Node* m = src_n->fast_out(j); // Get user
759 if (m == n) continue;
760 if (!m->is_Mach()) continue;
761 MachNode *mach = m->as_Mach();
762 bool src_matches = false;
763 int iop = mach->ideal_Opcode();
764
765 switch (iop) {
766 case Op_StoreB:
767 case Op_StoreC:
768 case Op_StoreD:
769 case Op_StoreF:
770 case Op_StoreI:
771 case Op_StoreL:
772 case Op_StoreLSpecial:
773 case Op_StoreP:
774 case Op_StoreN:
775 case Op_StoreVector:
776 case Op_StoreVectorMasked:
777 case Op_StoreVectorScatter:
778 case Op_StoreVectorScatterMasked:
779 case Op_StoreNKlass:
780 for (uint k = 1; k < m->req(); k++) {
781 Node *in = m->in(k);
782 if (in == src_n) {
783 src_matches = true;
784 break;
785 }
786 }
787 break;
788
789 default:
790 src_matches = true;
791 break;
792 }
919 // Children of projections are now all ready
920 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
921 Node* m = n->fast_out(j); // Get user
922 if(get_block_for_node(m) != block) {
923 continue;
924 }
925 if( m->is_Phi() ) continue;
926 int m_cnt = ready_cnt.at(m->_idx) - 1;
927 ready_cnt.at_put(m->_idx, m_cnt);
928 if( m_cnt == 0 )
929 worklist.push(m);
930 }
931
932 }
933
934 // Act as if the call defines the Frame Pointer.
935 // Certainly the FP is alive and well after the call.
936 regs.Insert(_matcher.c_frame_pointer());
937
938 // Set all registers killed and not already defined by the call.
939 uint r_cnt = mcall->tf()->range_cc()->cnt();
940 int op = mcall->ideal_Opcode();
941 MachProjNode *proj = new MachProjNode( mcall, r_cnt+1, RegMask::Empty, MachProjNode::fat_proj );
942 map_node_to_block(proj, block);
943 block->insert_node(proj, node_cnt++);
944
945 // Select the right register save policy.
946 const char *save_policy = nullptr;
947 switch (op) {
948 case Op_CallRuntime:
949 case Op_CallLeaf:
950 case Op_CallLeafNoFP:
951 case Op_CallLeafVector:
952 // Calling C code so use C calling convention
953 save_policy = _matcher._c_reg_save_policy;
954 break;
955
956 case Op_CallStaticJava:
957 case Op_CallDynamicJava:
958 // Calling Java code so use Java calling convention
959 save_policy = _matcher._register_save_policy;
|