177 case Op_LoadD:
178 case Op_LoadF:
179 case Op_LoadI:
180 case Op_LoadL:
181 case Op_LoadP:
182 case Op_LoadN:
183 case Op_LoadS:
184 case Op_LoadKlass:
185 case Op_LoadNKlass:
186 case Op_LoadRange:
187 case Op_LoadD_unaligned:
188 case Op_LoadL_unaligned:
189 assert(mach->in(2) == val, "should be address");
190 break;
191 case Op_StoreB:
192 case Op_StoreC:
193 case Op_StoreD:
194 case Op_StoreF:
195 case Op_StoreI:
196 case Op_StoreL:
197 case Op_StoreP:
198 case Op_StoreN:
199 case Op_StoreNKlass:
200 was_store = true; // Memory op is a store op
201 // Stores will have their address in slot 2 (memory in slot 1).
202 // If the value being nul-checked is in another slot, it means we
203 // are storing the checked value, which does NOT check the value!
204 if( mach->in(2) != val ) continue;
205 break; // Found a memory op?
206 case Op_StrComp:
207 case Op_StrEquals:
208 case Op_StrIndexOf:
209 case Op_StrIndexOfChar:
210 case Op_AryEq:
211 case Op_VectorizedHashCode:
212 case Op_StrInflatedCopy:
213 case Op_StrCompressedCopy:
214 case Op_EncodeISOArray:
215 case Op_CountPositives:
216 // Not a legit memory op for implicit null check regardless of
266 const Node* base = mach->get_base_and_disp(offset, adr_type);
267 if (base == nullptr || base == NodeSentinel) {
268 // Narrow oop address doesn't have base, only index.
269 // Give up if offset is beyond page size or if heap base is not protected.
270 if (val->bottom_type()->isa_narrowoop() &&
271 (MacroAssembler::needs_explicit_null_check(offset) ||
272 !CompressedOops::use_implicit_null_checks()))
273 continue;
274 // cannot reason about it; is probably not implicit null exception
275 } else {
276 const TypePtr* tptr;
277 if ((UseCompressedOops && CompressedOops::shift() == 0) ||
278 (UseCompressedClassPointers && CompressedKlassPointers::shift() == 0)) {
279 // 32-bits narrow oop can be the base of address expressions
280 tptr = base->get_ptr_type();
281 } else {
282 // only regular oops are expected here
283 tptr = base->bottom_type()->is_ptr();
284 }
285 // Give up if offset is not a compile-time constant.
286 if (offset == Type::OffsetBot || tptr->_offset == Type::OffsetBot)
287 continue;
288 offset += tptr->_offset; // correct if base is offsetted
289 // Give up if reference is beyond page size.
290 if (MacroAssembler::needs_explicit_null_check(offset))
291 continue;
292 // Give up if base is a decode node and the heap base is not protected.
293 if (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_DecodeN &&
294 !CompressedOops::use_implicit_null_checks())
295 continue;
296 }
297 }
298
299 // Check ctrl input to see if the null-check dominates the memory op
300 Block *cb = get_block_for_node(mach);
301 cb = cb->_idom; // Always hoist at least 1 block
302 if( !was_store ) { // Stores can be hoisted only one block
303 while( cb->_dom_depth > (block->_dom_depth + 1))
304 cb = cb->_idom; // Hoist loads as far as we want
305 // The non-null-block should dominate the memory op, too. Live
306 // range spilling will insert a spill in the non-null-block if it is
307 // needs to spill the memory op for an implicit null check.
308 if (cb->_dom_depth == (block->_dom_depth + 1)) {
309 if (cb != not_null_block) continue;
310 cb = cb->_idom;
311 }
312 }
313 if( cb != block ) continue;
314
315 // Found a memory user; see if it can be hoisted to check-block
316 uint vidx = 0; // Capture index of value into memop
317 uint j;
318 for( j = mach->req()-1; j > 0; j-- ) {
319 if( mach->in(j) == val ) {
320 vidx = j;
321 // Ignore DecodeN val which could be hoisted to where needed.
322 if( is_decoden ) continue;
323 }
324 // Block of memory-op input
325 Block *inb = get_block_for_node(mach->in(j));
326 Block *b = block; // Start from nul check
327 while( b != inb && b->_dom_depth > inb->_dom_depth )
328 b = b->_idom; // search upwards for input
329 // See if input dominates null check
330 if( b != inb )
331 break;
332 }
333 if( j > 0 )
334 continue;
335 Block *mb = get_block_for_node(mach);
336 // Hoisting stores requires more checks for the anti-dependence case.
337 // Give up hoisting if we have to move the store past any load.
338 if (was_store) {
339 // Make sure control does not do a merge (would have to check allpaths)
340 if (mb->num_preds() != 2) {
341 continue;
342 }
343 // mach is a store, hence block is the immediate dominator of mb.
344 // Due to the null-check shape of block (where its successors cannot re-join),
345 // block must be the direct predecessor of mb.
398 tempb->find_remove(temp);
399 block->add_inst(temp);
400 map_node_to_block(temp, block);
401 }
402 }
403 valb->find_remove(val);
404 block->add_inst(val);
405 map_node_to_block(val, block);
406 // DecodeN on x86 may kill flags. Check for flag-killing projections
407 // that also need to be hoisted.
408 for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) {
409 Node* n = val->fast_out(j);
410 if( n->is_MachProj() ) {
411 get_block_for_node(n)->find_remove(n);
412 block->add_inst(n);
413 map_node_to_block(n, block);
414 }
415 }
416 }
417 }
418 // Hoist the memory candidate up to the end of the test block.
419 Block *old_block = get_block_for_node(best);
420 old_block->find_remove(best);
421 block->add_inst(best);
422 map_node_to_block(best, block);
423
424 // Move the control dependence if it is pinned to not-null block.
425 // Don't change it in other cases: null or dominating control.
426 Node* ctrl = best->in(0);
427 if (ctrl != nullptr && get_block_for_node(ctrl) == not_null_block) {
428 // Set it to control edge of null check.
429 best->set_req(0, proj->in(0)->in(0));
430 }
431
432 // Check for flag-killing projections that also need to be hoisted
433 // Should be DU safe because no edge updates.
434 for (DUIterator_Fast jmax, j = best->fast_outs(jmax); j < jmax; j++) {
435 Node* n = best->fast_out(j);
436 if( n->is_MachProj() ) {
437 get_block_for_node(n)->find_remove(n);
711 if (src == 0) continue;
712 LRG& lrg_src = _regalloc->lrgs(src);
713 // detect if the live range ends or not
714 if (liveout->member(src) == false) {
715 lrg_ends = true;
716 for (DUIterator_Fast jmax, j = src_n->fast_outs(jmax); j < jmax; j++) {
717 Node* m = src_n->fast_out(j); // Get user
718 if (m == n) continue;
719 if (!m->is_Mach()) continue;
720 MachNode *mach = m->as_Mach();
721 bool src_matches = false;
722 int iop = mach->ideal_Opcode();
723
724 switch (iop) {
725 case Op_StoreB:
726 case Op_StoreC:
727 case Op_StoreD:
728 case Op_StoreF:
729 case Op_StoreI:
730 case Op_StoreL:
731 case Op_StoreP:
732 case Op_StoreN:
733 case Op_StoreVector:
734 case Op_StoreVectorMasked:
735 case Op_StoreVectorScatter:
736 case Op_StoreVectorScatterMasked:
737 case Op_StoreNKlass:
738 for (uint k = 1; k < m->req(); k++) {
739 Node *in = m->in(k);
740 if (in == src_n) {
741 src_matches = true;
742 break;
743 }
744 }
745 break;
746
747 default:
748 src_matches = true;
749 break;
750 }
871 // Children of projections are now all ready
872 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
873 Node* m = n->fast_out(j); // Get user
874 if(get_block_for_node(m) != block) {
875 continue;
876 }
877 if( m->is_Phi() ) continue;
878 int m_cnt = ready_cnt.at(m->_idx) - 1;
879 ready_cnt.at_put(m->_idx, m_cnt);
880 if( m_cnt == 0 )
881 worklist.push(m);
882 }
883
884 }
885
886 // Act as if the call defines the Frame Pointer.
887 // Certainly the FP is alive and well after the call.
888 regs.Insert(_matcher.c_frame_pointer());
889
890 // Set all registers killed and not already defined by the call.
891 uint r_cnt = mcall->tf()->range()->cnt();
892 int op = mcall->ideal_Opcode();
893 MachProjNode *proj = new MachProjNode( mcall, r_cnt+1, RegMask::Empty, MachProjNode::fat_proj );
894 map_node_to_block(proj, block);
895 block->insert_node(proj, node_cnt++);
896
897 // Select the right register save policy.
898 const char *save_policy = nullptr;
899 switch (op) {
900 case Op_CallRuntime:
901 case Op_CallLeaf:
902 case Op_CallLeafNoFP:
903 case Op_CallLeafVector:
904 // Calling C code so use C calling convention
905 save_policy = _matcher._c_reg_save_policy;
906 break;
907
908 case Op_CallStaticJava:
909 case Op_CallDynamicJava:
910 // Calling Java code so use Java calling convention
911 save_policy = _matcher._register_save_policy;
|
177 case Op_LoadD:
178 case Op_LoadF:
179 case Op_LoadI:
180 case Op_LoadL:
181 case Op_LoadP:
182 case Op_LoadN:
183 case Op_LoadS:
184 case Op_LoadKlass:
185 case Op_LoadNKlass:
186 case Op_LoadRange:
187 case Op_LoadD_unaligned:
188 case Op_LoadL_unaligned:
189 assert(mach->in(2) == val, "should be address");
190 break;
191 case Op_StoreB:
192 case Op_StoreC:
193 case Op_StoreD:
194 case Op_StoreF:
195 case Op_StoreI:
196 case Op_StoreL:
197 case Op_StoreLSpecial:
198 case Op_StoreP:
199 case Op_StoreN:
200 case Op_StoreNKlass:
201 was_store = true; // Memory op is a store op
202 // Stores will have their address in slot 2 (memory in slot 1).
203 // If the value being nul-checked is in another slot, it means we
204 // are storing the checked value, which does NOT check the value!
205 if( mach->in(2) != val ) continue;
206 break; // Found a memory op?
207 case Op_StrComp:
208 case Op_StrEquals:
209 case Op_StrIndexOf:
210 case Op_StrIndexOfChar:
211 case Op_AryEq:
212 case Op_VectorizedHashCode:
213 case Op_StrInflatedCopy:
214 case Op_StrCompressedCopy:
215 case Op_EncodeISOArray:
216 case Op_CountPositives:
217 // Not a legit memory op for implicit null check regardless of
267 const Node* base = mach->get_base_and_disp(offset, adr_type);
268 if (base == nullptr || base == NodeSentinel) {
269 // Narrow oop address doesn't have base, only index.
270 // Give up if offset is beyond page size or if heap base is not protected.
271 if (val->bottom_type()->isa_narrowoop() &&
272 (MacroAssembler::needs_explicit_null_check(offset) ||
273 !CompressedOops::use_implicit_null_checks()))
274 continue;
275 // cannot reason about it; is probably not implicit null exception
276 } else {
277 const TypePtr* tptr;
278 if ((UseCompressedOops && CompressedOops::shift() == 0) ||
279 (UseCompressedClassPointers && CompressedKlassPointers::shift() == 0)) {
280 // 32-bits narrow oop can be the base of address expressions
281 tptr = base->get_ptr_type();
282 } else {
283 // only regular oops are expected here
284 tptr = base->bottom_type()->is_ptr();
285 }
286 // Give up if offset is not a compile-time constant.
287 if (offset == Type::OffsetBot || tptr->offset() == Type::OffsetBot)
288 continue;
289 offset += tptr->offset(); // correct if base is offsetted
290 // Give up if reference is beyond page size.
291 if (MacroAssembler::needs_explicit_null_check(offset))
292 continue;
293 // Give up if base is a decode node and the heap base is not protected.
294 if (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_DecodeN &&
295 !CompressedOops::use_implicit_null_checks())
296 continue;
297 }
298 }
299
300 // Check ctrl input to see if the null-check dominates the memory op
301 Block *cb = get_block_for_node(mach);
302 cb = cb->_idom; // Always hoist at least 1 block
303 if( !was_store ) { // Stores can be hoisted only one block
304 while( cb->_dom_depth > (block->_dom_depth + 1))
305 cb = cb->_idom; // Hoist loads as far as we want
306 // The non-null-block should dominate the memory op, too. Live
307 // range spilling will insert a spill in the non-null-block if it is
308 // needs to spill the memory op for an implicit null check.
309 if (cb->_dom_depth == (block->_dom_depth + 1)) {
310 if (cb != not_null_block) continue;
311 cb = cb->_idom;
312 }
313 }
314 if( cb != block ) continue;
315
316 // Found a memory user; see if it can be hoisted to check-block
317 uint vidx = 0; // Capture index of value into memop
318 uint j;
319 for( j = mach->req()-1; j > 0; j-- ) {
320 if( mach->in(j) == val ) {
321 vidx = j;
322 // Ignore DecodeN val which could be hoisted to where needed.
323 if( is_decoden ) continue;
324 }
325 // Block of memory-op input
326 Block* inb = get_block_for_node(mach->in(j));
327 if (mach->in(j)->is_Con() && mach->in(j)->req() == 1 && inb == get_block_for_node(mach)) {
328 // Ignore constant loads scheduled in the same block (we can simply hoist them as well)
329 continue;
330 }
331 Block *b = block; // Start from nul check
332 while( b != inb && b->_dom_depth > inb->_dom_depth )
333 b = b->_idom; // search upwards for input
334 // See if input dominates null check
335 if( b != inb )
336 break;
337 }
338 if( j > 0 )
339 continue;
340 Block *mb = get_block_for_node(mach);
341 // Hoisting stores requires more checks for the anti-dependence case.
342 // Give up hoisting if we have to move the store past any load.
343 if (was_store) {
344 // Make sure control does not do a merge (would have to check allpaths)
345 if (mb->num_preds() != 2) {
346 continue;
347 }
348 // mach is a store, hence block is the immediate dominator of mb.
349 // Due to the null-check shape of block (where its successors cannot re-join),
350 // block must be the direct predecessor of mb.
403 tempb->find_remove(temp);
404 block->add_inst(temp);
405 map_node_to_block(temp, block);
406 }
407 }
408 valb->find_remove(val);
409 block->add_inst(val);
410 map_node_to_block(val, block);
411 // DecodeN on x86 may kill flags. Check for flag-killing projections
412 // that also need to be hoisted.
413 for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) {
414 Node* n = val->fast_out(j);
415 if( n->is_MachProj() ) {
416 get_block_for_node(n)->find_remove(n);
417 block->add_inst(n);
418 map_node_to_block(n, block);
419 }
420 }
421 }
422 }
423
424 // Hoist constant load inputs as well.
425 for (uint i = 1; i < best->req(); ++i) {
426 Node* n = best->in(i);
427 if (n->is_Con() && get_block_for_node(n) == get_block_for_node(best)) {
428 get_block_for_node(n)->find_remove(n);
429 block->add_inst(n);
430 map_node_to_block(n, block);
431 // Constant loads may kill flags (for example, when XORing a register).
432 // Check for flag-killing projections that also need to be hoisted.
433 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
434 Node* proj = n->fast_out(j);
435 if (proj->is_MachProj()) {
436 get_block_for_node(proj)->find_remove(proj);
437 block->add_inst(proj);
438 map_node_to_block(proj, block);
439 }
440 }
441 }
442 }
443
444 // Hoist the memory candidate up to the end of the test block.
445 Block *old_block = get_block_for_node(best);
446 old_block->find_remove(best);
447 block->add_inst(best);
448 map_node_to_block(best, block);
449
450 // Move the control dependence if it is pinned to not-null block.
451 // Don't change it in other cases: null or dominating control.
452 Node* ctrl = best->in(0);
453 if (ctrl != nullptr && get_block_for_node(ctrl) == not_null_block) {
454 // Set it to control edge of null check.
455 best->set_req(0, proj->in(0)->in(0));
456 }
457
458 // Check for flag-killing projections that also need to be hoisted
459 // Should be DU safe because no edge updates.
460 for (DUIterator_Fast jmax, j = best->fast_outs(jmax); j < jmax; j++) {
461 Node* n = best->fast_out(j);
462 if( n->is_MachProj() ) {
463 get_block_for_node(n)->find_remove(n);
737 if (src == 0) continue;
738 LRG& lrg_src = _regalloc->lrgs(src);
739 // detect if the live range ends or not
740 if (liveout->member(src) == false) {
741 lrg_ends = true;
742 for (DUIterator_Fast jmax, j = src_n->fast_outs(jmax); j < jmax; j++) {
743 Node* m = src_n->fast_out(j); // Get user
744 if (m == n) continue;
745 if (!m->is_Mach()) continue;
746 MachNode *mach = m->as_Mach();
747 bool src_matches = false;
748 int iop = mach->ideal_Opcode();
749
750 switch (iop) {
751 case Op_StoreB:
752 case Op_StoreC:
753 case Op_StoreD:
754 case Op_StoreF:
755 case Op_StoreI:
756 case Op_StoreL:
757 case Op_StoreLSpecial:
758 case Op_StoreP:
759 case Op_StoreN:
760 case Op_StoreVector:
761 case Op_StoreVectorMasked:
762 case Op_StoreVectorScatter:
763 case Op_StoreVectorScatterMasked:
764 case Op_StoreNKlass:
765 for (uint k = 1; k < m->req(); k++) {
766 Node *in = m->in(k);
767 if (in == src_n) {
768 src_matches = true;
769 break;
770 }
771 }
772 break;
773
774 default:
775 src_matches = true;
776 break;
777 }
898 // Children of projections are now all ready
899 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
900 Node* m = n->fast_out(j); // Get user
901 if(get_block_for_node(m) != block) {
902 continue;
903 }
904 if( m->is_Phi() ) continue;
905 int m_cnt = ready_cnt.at(m->_idx) - 1;
906 ready_cnt.at_put(m->_idx, m_cnt);
907 if( m_cnt == 0 )
908 worklist.push(m);
909 }
910
911 }
912
913 // Act as if the call defines the Frame Pointer.
914 // Certainly the FP is alive and well after the call.
915 regs.Insert(_matcher.c_frame_pointer());
916
917 // Set all registers killed and not already defined by the call.
918 uint r_cnt = mcall->tf()->range_cc()->cnt();
919 int op = mcall->ideal_Opcode();
920 MachProjNode *proj = new MachProjNode( mcall, r_cnt+1, RegMask::Empty, MachProjNode::fat_proj );
921 map_node_to_block(proj, block);
922 block->insert_node(proj, node_cnt++);
923
924 // Select the right register save policy.
925 const char *save_policy = nullptr;
926 switch (op) {
927 case Op_CallRuntime:
928 case Op_CallLeaf:
929 case Op_CallLeafNoFP:
930 case Op_CallLeafVector:
931 // Calling C code so use C calling convention
932 save_policy = _matcher._c_reg_save_policy;
933 break;
934
935 case Op_CallStaticJava:
936 case Op_CallDynamicJava:
937 // Calling Java code so use Java calling convention
938 save_policy = _matcher._register_save_policy;
|