< prev index next >

src/hotspot/share/opto/loopopts.cpp

Print this page
@@ -31,10 +31,11 @@
  #include "opto/callnode.hpp"
  #include "opto/castnode.hpp"
  #include "opto/connode.hpp"
  #include "opto/castnode.hpp"
  #include "opto/divnode.hpp"
+ #include "opto/inlinetypenode.hpp"
  #include "opto/loopnode.hpp"
  #include "opto/matcher.hpp"
  #include "opto/mulnode.hpp"
  #include "opto/movenode.hpp"
  #include "opto/opaquenode.hpp"

@@ -59,10 +60,16 @@
    if (n->Opcode() == Op_CastII && region->is_CountedLoop() &&
        n->in(1) == region->as_CountedLoop()->phi()) {
      return NULL;
    }
  
+   // Inline types should not be split through Phis because they cannot be merged
+   // through Phi nodes but each value input needs to be merged individually.
+   if (n->is_InlineType()) {
+     return NULL;
+   }
+ 
    // Bail out if 'n' is a Div or Mod node whose zero check was removed earlier (i.e. control is NULL) and its divisor is an induction variable
    // phi p of a trip-counted (integer) loop whose inputs could be zero (include zero in their type range). p could have a more precise type
    // range that does not necessarily include all values of its inputs. Since each of these inputs will be a divisor of the newly cloned nodes
    // of 'n', we need to bail out of one of these divisors could be zero (zero in its type range).
    if ((n->Opcode() == Op_DivI || n->Opcode() == Op_ModI) && n->in(0) == NULL

@@ -974,10 +981,59 @@
        }
      }
    }
  }
  
+ // If UseArrayMarkWordCheck is enabled, we can't use immutable memory for the flat array check
+ // because we are loading the mark word which is mutable. Although the bits we are interested in
+ // are immutable (we check for markWord::unlocked_value), we need to use raw memory to not break
+ // anti dependency analysis. Below code will attempt to still move flat array checks out of loops,
+ // mainly to enable loop unswitching.
+ void PhaseIdealLoop::move_flat_array_check_out_of_loop(Node* n) {
+   // Skip checks for more than one array
+   if (n->req() > 3) {
+     return;
+   }
+   Node* mem = n->in(FlatArrayCheckNode::Memory);
+   Node* array = n->in(FlatArrayCheckNode::Array)->uncast();
+   IdealLoopTree* check_loop = get_loop(get_ctrl(n));
+   IdealLoopTree* ary_loop = get_loop(get_ctrl(array));
+ 
+   // Check if array is loop invariant
+   if (!check_loop->is_member(ary_loop)) {
+     // Walk up memory graph from the check until we leave the loop
+     VectorSet wq;
+     wq.set(mem->_idx);
+     while (check_loop->is_member(get_loop(ctrl_or_self(mem)))) {
+       if (mem->is_Phi()) {
+         mem = mem->in(1);
+       } else if (mem->is_MergeMem()) {
+         mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
+       } else if (mem->is_Proj()) {
+         mem = mem->in(0);
+       } else if (mem->is_MemBar() || mem->is_SafePoint()) {
+         mem = mem->in(TypeFunc::Memory);
+       } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
+         mem = mem->in(MemNode::Memory);
+       } else {
+ #ifdef ASSERT
+         mem->dump();
+ #endif
+         ShouldNotReachHere();
+       }
+       if (wq.test_set(mem->_idx)) {
+         return;
+       }
+     }
+     // Replace memory input and re-compute ctrl to move the check out of the loop
+     _igvn.replace_input_of(n, 1, mem);
+     set_ctrl_and_loop(n, get_early_ctrl(n));
+     Node* bol = n->unique_out();
+     set_ctrl_and_loop(bol, get_early_ctrl(bol));
+   }
+ }
+ 
  //------------------------------split_if_with_blocks_pre-----------------------
  // Do the real work in a non-recursive function.  Data nodes want to be
  // cloned in the pre-order so they can feed each other nicely.
  Node *PhaseIdealLoop::split_if_with_blocks_pre( Node *n ) {
    // Cloning these guys is unlikely to win

@@ -986,10 +1042,16 @@
      return n;
    }
    if (n->is_Proj()) {
      return n;
    }
+ 
+   if (UseArrayMarkWordCheck && n->isa_FlatArrayCheck()) {
+     move_flat_array_check_out_of_loop(n);
+     return n;
+   }
+ 
    // Do not clone-up CmpFXXX variations, as these are always
    // followed by a CmpI
    if (n->is_Cmp()) {
      return n;
    }

@@ -1258,16 +1320,116 @@
    }
  
    return out_le;
  }
  
+ bool PhaseIdealLoop::flatten_array_element_type_check(Node *n) {
+   // If the CmpP is a subtype check for a value that has just been
+   // loaded from an array, the subtype check guarantees the value
+   // can't be stored in a flattened array and the load of the value
+   // happens with a flattened array check then: push the type check
+   // through the phi of the flattened array check. This needs special
+   // logic because the subtype check's input is not a phi but a
+   // LoadKlass that must first be cloned through the phi.
+   if (n->Opcode() != Op_CmpP) {
+     return false;
+   }
+ 
+   Node* klassptr = n->in(1);
+   Node* klasscon = n->in(2);
+ 
+   if (klassptr->is_DecodeNarrowPtr()) {
+     klassptr = klassptr->in(1);
+   }
+ 
+   if (klassptr->Opcode() != Op_LoadKlass && klassptr->Opcode() != Op_LoadNKlass) {
+     return false;
+   }
+ 
+   if (!klasscon->is_Con()) {
+     return false;
+   }
+ 
+   Node* addr = klassptr->in(MemNode::Address);
+ 
+   if (!addr->is_AddP()) {
+     return false;
+   }
+ 
+   intptr_t offset;
+   Node* obj = AddPNode::Ideal_base_and_offset(addr, &_igvn, offset);
+ 
+   if (obj == NULL) {
+     return false;
+   }
+ 
+   assert(obj != NULL && addr->in(AddPNode::Base) == addr->in(AddPNode::Address), "malformed AddP?");
+   if (obj->Opcode() == Op_CastPP) {
+     obj = obj->in(1);
+   }
+ 
+   if (!obj->is_Phi()) {
+     return false;
+   }
+ 
+   Node* region = obj->in(0);
+ 
+   Node* phi = PhiNode::make_blank(region, n->in(1));
+   for (uint i = 1; i < region->req(); i++) {
+     Node* in = obj->in(i);
+     Node* ctrl = region->in(i);
+     if (addr->in(AddPNode::Base) != obj) {
+       Node* cast = addr->in(AddPNode::Base);
+       assert(cast->Opcode() == Op_CastPP && cast->in(0) != NULL, "inconsistent subgraph");
+       Node* cast_clone = cast->clone();
+       cast_clone->set_req(0, ctrl);
+       cast_clone->set_req(1, in);
+       register_new_node(cast_clone, ctrl);
+       _igvn.set_type(cast_clone, cast_clone->Value(&_igvn));
+       in = cast_clone;
+     }
+     Node* addr_clone = addr->clone();
+     addr_clone->set_req(AddPNode::Base, in);
+     addr_clone->set_req(AddPNode::Address, in);
+     register_new_node(addr_clone, ctrl);
+     _igvn.set_type(addr_clone, addr_clone->Value(&_igvn));
+     Node* klassptr_clone = klassptr->clone();
+     klassptr_clone->set_req(2, addr_clone);
+     register_new_node(klassptr_clone, ctrl);
+     _igvn.set_type(klassptr_clone, klassptr_clone->Value(&_igvn));
+     if (klassptr != n->in(1)) {
+       Node* decode = n->in(1);
+       assert(decode->is_DecodeNarrowPtr(), "inconsistent subgraph");
+       Node* decode_clone = decode->clone();
+       decode_clone->set_req(1, klassptr_clone);
+       register_new_node(decode_clone, ctrl);
+       _igvn.set_type(decode_clone, decode_clone->Value(&_igvn));
+       klassptr_clone = decode_clone;
+     }
+     phi->set_req(i, klassptr_clone);
+   }
+   register_new_node(phi, region);
+   Node* orig = n->in(1);
+   _igvn.replace_input_of(n, 1, phi);
+   split_if_with_blocks_post(n);
+   if (n->outcnt() != 0) {
+     _igvn.replace_input_of(n, 1, orig);
+     _igvn.remove_dead_node(phi);
+   }
+   return true;
+ }
+ 
  //------------------------------split_if_with_blocks_post----------------------
  // Do the real work in a non-recursive function.  CFG hackery wants to be
  // in the post-order, so it can dirty the I-DOM info and not use the dirtied
  // info.
  void PhaseIdealLoop::split_if_with_blocks_post(Node *n) {
  
+   if (flatten_array_element_type_check(n)) {
+     return;
+   }
+ 
    // Cloning Cmp through Phi's involves the split-if transform.
    // FastLock is not used by an If
    if (n->is_Cmp() && !n->is_FastLock()) {
      Node *n_ctrl = get_ctrl(n);
      // Determine if the Node has inputs from some local Phi.

@@ -1419,10 +1581,15 @@
  
    try_sink_out_of_loop(n);
  
    try_move_store_after_loop(n);
  
+   // Remove multiple allocations of the same inline type
+   if (n->is_InlineType()) {
+     n->as_InlineType()->remove_redundant_allocations(&_igvn, this);
+   }
+ 
    // Check for Opaque2's who's loop has disappeared - who's input is in the
    // same loop nest as their output.  Remove 'em, they are no longer useful.
    if( n_op == Op_Opaque2 &&
        n->in(1) != NULL &&
        get_loop(get_ctrl(n)) == get_loop(get_ctrl(n->in(1))) ) {

@@ -1727,14 +1894,22 @@
      sample_bool = n->in(1);
      assert(sample_bool->is_Bool(), "wrong type");
    } else {
      sample_bool = n;
    }
-   Node *sample_cmp = sample_bool->in(1);
+   Node* sample_cmp = sample_bool->in(1);
+   const Type* t = Type::TOP;
+   const TypePtr* at = NULL;
+   if (sample_cmp->is_FlatArrayCheck()) {
+     // Left input of a FlatArrayCheckNode is memory, set the (adr) type of the phi accordingly
+     assert(sample_cmp->in(1)->bottom_type() == Type::MEMORY, "unexpected input type");
+     t = Type::MEMORY;
+     at = TypeRawPtr::BOTTOM;
+   }
  
    // Make Phis to merge the Cmp's inputs.
-   PhiNode *phi1 = new PhiNode(phi->in(0), Type::TOP);
+   PhiNode *phi1 = new PhiNode(phi->in(0), t, at);
    PhiNode *phi2 = new PhiNode(phi->in(0), Type::TOP);
    for (i = 1; i < phi->req(); i++) {
      Node *n1 = sample_opaque == NULL ? phi->in(i)->in(1)->in(1) : phi->in(i)->in(1)->in(1)->in(1);
      Node *n2 = sample_opaque == NULL ? phi->in(i)->in(1)->in(2) : phi->in(i)->in(1)->in(1)->in(2);
      phi1->set_req(i, n1);
< prev index next >