< prev index next >

src/hotspot/share/opto/parse2.cpp

Print this page
@@ -23,10 +23,11 @@
   */
  
  #include "precompiled.hpp"
  #include "jvm_io.h"
  #include "ci/ciMethodData.hpp"
+ #include "ci/ciSymbols.hpp"
  #include "classfile/vmSymbols.hpp"
  #include "compiler/compileLog.hpp"
  #include "interpreter/linkResolver.hpp"
  #include "memory/resourceArea.hpp"
  #include "memory/universe.hpp"

@@ -34,10 +35,12 @@
  #include "opto/addnode.hpp"
  #include "opto/castnode.hpp"
  #include "opto/convertnode.hpp"
  #include "opto/divnode.hpp"
  #include "opto/idealGraphPrinter.hpp"
+ #include "opto/idealKit.hpp"
+ #include "opto/inlinetypenode.hpp"
  #include "opto/matcher.hpp"
  #include "opto/memnode.hpp"
  #include "opto/mulnode.hpp"
  #include "opto/opaquenode.hpp"
  #include "opto/parse.hpp"

@@ -48,62 +51,291 @@
  #ifndef PRODUCT
  extern int explicit_null_checks_inserted,
             explicit_null_checks_elided;
  #endif
  
+ Node* Parse::record_profile_for_speculation_at_array_load(Node* ld) {
+   // Feed unused profile data to type speculation
+   if (UseTypeSpeculation && UseArrayLoadStoreProfile) {
+     ciKlass* array_type = NULL;
+     ciKlass* element_type = NULL;
+     ProfilePtrKind element_ptr = ProfileMaybeNull;
+     bool flat_array = true;
+     bool null_free_array = true;
+     method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
+     if (element_type != NULL || element_ptr != ProfileMaybeNull) {
+       ld = record_profile_for_speculation(ld, element_type, element_ptr);
+     }
+   }
+   return ld;
+ }
+ 
+ 
  //---------------------------------array_load----------------------------------
  void Parse::array_load(BasicType bt) {
    const Type* elemtype = Type::TOP;
-   bool big_val = bt == T_DOUBLE || bt == T_LONG;
    Node* adr = array_addressing(bt, 0, elemtype);
    if (stopped())  return;     // guaranteed null or range check
  
-   pop();                      // index (already used)
-   Node* array = pop();        // the array itself
+   Node* idx = pop();
+   Node* ary = pop();
+ 
+   // Handle inline type arrays
+   const TypeOopPtr* elemptr = elemtype->make_oopptr();
+   const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr();
+   if (ary_t->is_flat()) {
+     // Load from flattened inline type array
+     Node* vt = InlineTypeNode::make_from_flattened(this, elemtype->inline_klass(), ary, adr);
+     push(vt);
+     return;
+   } else if (ary_t->is_null_free()) {
+     // Load from non-flattened inline type array (elements can never be null)
+     bt = T_INLINE_TYPE;
+   } else if (!ary_t->is_not_flat()) {
+     // Cannot statically determine if array is flattened, emit runtime check
+     assert(UseFlatArray && is_reference_type(bt) && elemptr->can_be_inline_type() && !ary_t->klass_is_exact() && !ary_t->is_not_null_free() &&
+            (!elemptr->is_inlinetypeptr() || elemptr->inline_klass()->flatten_array()), "array can't be flattened");
+     IdealKit ideal(this);
+     IdealVariable res(ideal);
+     ideal.declarations_done();
+     ideal.if_then(flat_array_test(ary, /* flat = */ false)); {
+       // non-flattened
+       assert(ideal.ctrl()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found");
+       sync_kit(ideal);
+       const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);
+       Node* ld = access_load_at(ary, adr, adr_type, elemptr, bt,
+                                 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD);
+       if (elemptr->is_inlinetypeptr()) {
+         assert(elemptr->maybe_null(), "null free array should be handled above");
+         ld = InlineTypeNode::make_from_oop(this, ld, elemptr->inline_klass(), false);
+       }
+       ideal.sync_kit(this);
+       ideal.set(res, ld);
+     } ideal.else_(); {
+       // flattened
+       sync_kit(ideal);
+       if (elemptr->is_inlinetypeptr()) {
+         // Element type is known, cast and load from flattened representation
+         ciInlineKlass* vk = elemptr->inline_klass();
+         assert(vk->flatten_array() && elemptr->maybe_null(), "never/always flat - should be optimized");
+         ciArrayKlass* array_klass = ciArrayKlass::make(vk, /* null_free */ true);
+         const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr();
+         Node* cast = _gvn.transform(new CheckCastPPNode(control(), ary, arytype));
+         Node* casted_adr = array_element_address(cast, idx, T_INLINE_TYPE, ary_t->size(), control());
+         // Re-execute flattened array load if buffering triggers deoptimization
+         PreserveReexecuteState preexecs(this);
+         jvms()->set_should_reexecute(true);
+         inc_sp(2);
+         Node* vt = InlineTypeNode::make_from_flattened(this, vk, cast, casted_adr)->buffer(this, false);
+         ideal.set(res, vt);
+         ideal.sync_kit(this);
+       } else {
+         // Element type is unknown, emit runtime call
+ 
+         // Below membars keep this access to an unknown flattened array correctly
+         // ordered with other unknown and known flattened array accesses.
+         insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
+ 
+         Node* call = NULL;
+         {
+           // Re-execute flattened array load if runtime call triggers deoptimization
+           PreserveReexecuteState preexecs(this);
+           jvms()->set_bci(_bci);
+           jvms()->set_should_reexecute(true);
+           inc_sp(2);
+           kill_dead_locals();
+           call = make_runtime_call(RC_NO_LEAF | RC_NO_IO,
+                                    OptoRuntime::load_unknown_inline_type(),
+                                    OptoRuntime::load_unknown_inline_Java(),
+                                    NULL, TypeRawPtr::BOTTOM,
+                                    ary, idx);
+         }
+         make_slow_call_ex(call, env()->Throwable_klass(), false);
+         Node* buffer = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
+ 
+         insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
+ 
+         // Keep track of the information that the inline type is flattened in arrays
+         const Type* unknown_value = elemptr->is_instptr()->cast_to_flatten_array();
+         buffer = _gvn.transform(new CheckCastPPNode(control(), buffer, unknown_value));
+ 
+         ideal.sync_kit(this);
+         ideal.set(res, buffer);
+       }
+     } ideal.end_if();
+     sync_kit(ideal);
+     Node* ld = _gvn.transform(ideal.value(res));
+     ld = record_profile_for_speculation_at_array_load(ld);
+     push_node(bt, ld);
+     return;
+   }
  
    if (elemtype == TypeInt::BOOL) {
      bt = T_BOOLEAN;
    }
    const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);
- 
-   Node* ld = access_load_at(array, adr, adr_type, elemtype, bt,
+   Node* ld = access_load_at(ary, adr, adr_type, elemtype, bt,
                              IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD);
-   if (big_val) {
-     push_pair(ld);
-   } else {
-     push(ld);
+   ld = record_profile_for_speculation_at_array_load(ld);
+   // Loading a non-flattened inline type
+   if (elemptr != NULL && elemptr->is_inlinetypeptr()) {
+     assert(!ary_t->is_null_free() || !elemptr->maybe_null(), "inline type array elements should never be null");
+     ld = InlineTypeNode::make_from_oop(this, ld, elemptr->inline_klass(), !elemptr->maybe_null());
    }
+   push_node(bt, ld);
  }
  
  
  //--------------------------------array_store----------------------------------
  void Parse::array_store(BasicType bt) {
    const Type* elemtype = Type::TOP;
-   bool big_val = bt == T_DOUBLE || bt == T_LONG;
-   Node* adr = array_addressing(bt, big_val ? 2 : 1, elemtype);
+   Node* adr = array_addressing(bt, type2size[bt], elemtype);
    if (stopped())  return;     // guaranteed null or range check
+   Node* cast_val = NULL;
    if (bt == T_OBJECT) {
-     array_store_check();
-     if (stopped()) {
-       return;
-     }
+     cast_val = array_store_check(adr, elemtype);
+     if (stopped()) return;
    }
-   Node* val;                  // Oop to store
-   if (big_val) {
-     val = pop_pair();
-   } else {
-     val = pop();
-   }
-   pop();                      // index (already used)
-   Node* array = pop();        // the array itself
+   Node* val = pop_node(bt); // Value to store
+   Node* idx = pop();        // Index in the array
+   Node* ary = pop();        // The array itself
+ 
+   const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr();
+   const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);
+   assert(adr->as_AddP()->in(AddPNode::Base) == ary, "inconsistent address base");
  
    if (elemtype == TypeInt::BOOL) {
      bt = T_BOOLEAN;
+   } else if (bt == T_OBJECT) {
+     elemtype = elemtype->make_oopptr();
+     const Type* tval = _gvn.type(cast_val);
+     // We may have lost type information for 'val' here due to the casts
+     // emitted by the array_store_check code (see JDK-6312651)
+     // TODO Remove this code once JDK-6312651 is in.
+     const Type* tval_init = _gvn.type(val);
+     // Based on the value to be stored, try to determine if the array is not null-free and/or not flat.
+     // This is only legal for non-null stores because the array_store_check always passes for null, even
+     // if the array is null-free. Null stores are handled in GraphKit::gen_inline_array_null_guard().
+     bool not_inline = !tval->isa_inlinetype() &&
+                       ((!tval_init->maybe_null() && !tval_init->is_oopptr()->can_be_inline_type()) ||
+                        (!tval->maybe_null() && !tval->is_oopptr()->can_be_inline_type()));
+     bool not_flattened = not_inline || ((tval_init->is_inlinetypeptr() || tval_init->isa_inlinetype()) && !tval_init->inline_klass()->flatten_array());
+     if (!ary_t->is_not_null_free() && not_inline) {
+       // Storing a non-inline type, mark array as not null-free (-> not flat).
+       ary_t = ary_t->cast_to_not_null_free();
+       Node* cast = _gvn.transform(new CheckCastPPNode(control(), ary, ary_t));
+       replace_in_map(ary, cast);
+       ary = cast;
+     } else if (!ary_t->is_not_flat() && not_flattened) {
+       // Storing a non-flattened value, mark array as not flat.
+       ary_t = ary_t->cast_to_not_flat();
+       Node* cast = _gvn.transform(new CheckCastPPNode(control(), ary, ary_t));
+       replace_in_map(ary, cast);
+       ary = cast;
+     }
+ 
+     if (ary_t->is_flat()) {
+       // Store to flattened inline type array
+       assert(!tval->maybe_null(), "should be guaranteed by array store check");
+       // Re-execute flattened array store if buffering triggers deoptimization
+       PreserveReexecuteState preexecs(this);
+       inc_sp(3);
+       jvms()->set_should_reexecute(true);
+       cast_val->as_InlineTypeBase()->store_flattened(this, ary, adr, NULL, 0, MO_UNORDERED | IN_HEAP | IS_ARRAY);
+       return;
+     } else if (ary_t->is_null_free()) {
+       // Store to non-flattened inline type array (elements can never be null)
+       assert(!tval->maybe_null(), "should be guaranteed by array store check");
+       if (elemtype->inline_klass()->is_empty()) {
+         // Ignore empty inline stores, array is already initialized.
+         return;
+       }
+     } else if (!ary_t->is_not_flat() && (tval != TypePtr::NULL_PTR || StressReflectiveCode)) {
+       // Array might be flattened, emit runtime checks (for NULL, a simple inline_array_null_guard is sufficient).
+       assert(UseFlatArray && !not_flattened && elemtype->is_oopptr()->can_be_inline_type() &&
+              !ary_t->klass_is_exact() && !ary_t->is_not_null_free(), "array can't be flattened");
+       IdealKit ideal(this);
+       ideal.if_then(flat_array_test(ary, /* flat = */ false)); {
+         // non-flattened
+         assert(ideal.ctrl()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found");
+         sync_kit(ideal);
+         Node* cast_ary = inline_array_null_guard(ary, cast_val, 3);
+         inc_sp(3);
+         access_store_at(cast_ary, adr, adr_type, cast_val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY, false);
+         dec_sp(3);
+         ideal.sync_kit(this);
+       } ideal.else_(); {
+         sync_kit(ideal);
+         Node* val = cast_val;
+         // flattened
+         if (!val->is_InlineType() && tval->maybe_null()) {
+           // Add null check
+           Node* null_ctl = top();
+           val = null_check_oop(val, &null_ctl);
+           if (null_ctl != top()) {
+             PreserveJVMState pjvms(this);
+             inc_sp(3);
+             set_control(null_ctl);
+             uncommon_trap(Deoptimization::Reason_null_check, Deoptimization::Action_none);
+             dec_sp(3);
+           }
+         }
+         // Try to determine the inline klass
+         ciInlineKlass* vk = NULL;
+         if (tval->isa_inlinetype() || tval->is_inlinetypeptr()) {
+           vk = tval->inline_klass();
+         } else if (tval_init->isa_inlinetype() || tval_init->is_inlinetypeptr()) {
+           vk = tval_init->inline_klass();
+         } else if (elemtype->is_inlinetypeptr()) {
+           vk = elemtype->inline_klass();
+         }
+         Node* casted_ary = ary;
+         if (vk != NULL && !stopped()) {
+           // Element type is known, cast and store to flattened representation
+           assert(vk->flatten_array() && elemtype->maybe_null(), "never/always flat - should be optimized");
+           ciArrayKlass* array_klass = ciArrayKlass::make(vk, /* null_free */ true);
+           const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr();
+           casted_ary = _gvn.transform(new CheckCastPPNode(control(), casted_ary, arytype));
+           Node* casted_adr = array_element_address(casted_ary, idx, T_OBJECT, arytype->size(), control());
+           if (!val->is_InlineType()) {
+             assert(!gvn().type(val)->maybe_null(), "inline type array elements should never be null");
+             val = InlineTypeNode::make_from_oop(this, val, vk);
+           }
+           // Re-execute flattened array store if buffering triggers deoptimization
+           PreserveReexecuteState preexecs(this);
+           inc_sp(3);
+           jvms()->set_should_reexecute(true);
+           val->as_InlineTypeBase()->store_flattened(this, casted_ary, casted_adr, NULL, 0, MO_UNORDERED | IN_HEAP | IS_ARRAY);
+         } else if (!stopped()) {
+           // Element type is unknown, emit runtime call
+ 
+           // Below membars keep this access to an unknown flattened array correctly
+           // ordered with other unknown and known flattened array accesses.
+           insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
+ 
+           make_runtime_call(RC_LEAF,
+                             OptoRuntime::store_unknown_inline_type(),
+                             CAST_FROM_FN_PTR(address, OptoRuntime::store_unknown_inline),
+                             "store_unknown_inline", TypeRawPtr::BOTTOM,
+                             val, casted_ary, idx);
+ 
+           insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
+         }
+         ideal.sync_kit(this);
+       }
+       ideal.end_if();
+       sync_kit(ideal);
+       return;
+     } else if (!ary_t->is_not_null_free()) {
+       // Array is not flattened but may be null free
+       assert(elemtype->is_oopptr()->can_be_inline_type() && !ary_t->klass_is_exact(), "array can't be null-free");
+       ary = inline_array_null_guard(ary, cast_val, 3, true);
+     }
    }
-   const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);
- 
-   access_store_at(array, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY);
+   inc_sp(3);
+   access_store_at(ary, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY);
+   dec_sp(3);
  }
  
  
  //------------------------------array_addressing-------------------------------
  // Pull array and index from the stack.  Compute pointer-to-element.

@@ -199,10 +431,126 @@
      }
    }
    // Check for always knowing you are throwing a range-check exception
    if (stopped())  return top();
  
+   // This could be an access to an inline type array. We can't tell if it's
+   // flat or not. Knowing the exact type avoids runtime checks and leads to
+   // a much simpler graph shape. Check profile information.
+   if (!arytype->is_flat() && !arytype->is_not_flat()) {
+     // First check the speculative type
+     Deoptimization::DeoptReason reason = Deoptimization::Reason_speculate_class_check;
+     ciKlass* array_type = arytype->speculative_type();
+     if (too_many_traps_or_recompiles(reason) || array_type == NULL) {
+       // No speculative type, check profile data at this bci
+       array_type = NULL;
+       reason = Deoptimization::Reason_class_check;
+       if (UseArrayLoadStoreProfile && !too_many_traps_or_recompiles(reason)) {
+         ciKlass* element_type = NULL;
+         ProfilePtrKind element_ptr = ProfileMaybeNull;
+         bool flat_array = true;
+         bool null_free_array = true;
+         method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
+       }
+     }
+     if (array_type != NULL) {
+       // Speculate that this array has the exact type reported by profile data
+       Node* better_ary = NULL;
+       DEBUG_ONLY(Node* old_control = control();)
+       Node* slow_ctl = type_check_receiver(ary, array_type, 1.0, &better_ary);
+       if (stopped()) {
+         // The check always fails and therefore profile information is incorrect. Don't use it.
+         assert(old_control == slow_ctl, "type check should have been removed");
+         set_control(slow_ctl);
+       } else if (!slow_ctl->is_top()) {
+         { PreserveJVMState pjvms(this);
+           set_control(slow_ctl);
+           uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
+         }
+         replace_in_map(ary, better_ary);
+         ary = better_ary;
+         arytype  = _gvn.type(ary)->is_aryptr();
+         elemtype = arytype->elem();
+       }
+     }
+   } else if (UseTypeSpeculation && UseArrayLoadStoreProfile) {
+     // No need to speculate: feed profile data at this bci for the
+     // array to type speculation
+     ciKlass* array_type = NULL;
+     ciKlass* element_type = NULL;
+     ProfilePtrKind element_ptr = ProfileMaybeNull;
+     bool flat_array = true;
+     bool null_free_array = true;
+     method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
+     if (array_type != NULL) {
+       ary = record_profile_for_speculation(ary, array_type, ProfileMaybeNull);
+     }
+   }
+ 
+   // We have no exact array type from profile data. Check profile data
+   // for a non null-free or non flat array. Non null-free implies non
+   // flat so check this one first. Speculating on a non null-free
+   // array doesn't help aaload but could be profitable for a
+   // subsequent aastore.
+   if (!arytype->is_null_free() && !arytype->is_not_null_free()) {
+     bool null_free_array = true;
+     Deoptimization::DeoptReason reason = Deoptimization::Reason_none;
+     if (arytype->speculative() != NULL &&
+         arytype->speculative()->is_aryptr()->is_not_null_free() &&
+         !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
+       null_free_array = false;
+       reason = Deoptimization::Reason_speculate_class_check;
+     } else if (UseArrayLoadStoreProfile && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) {
+       ciKlass* array_type = NULL;
+       ciKlass* element_type = NULL;
+       ProfilePtrKind element_ptr = ProfileMaybeNull;
+       bool flat_array = true;
+       method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
+       reason = Deoptimization::Reason_class_check;
+     }
+     if (!null_free_array) {
+       { // Deoptimize if null-free array
+         BuildCutout unless(this, null_free_array_test(load_object_klass(ary), /* null_free = */ false), PROB_MAX);
+         uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
+       }
+       assert(!stopped(), "null-free array should have been caught earlier");
+       Node* better_ary = _gvn.transform(new CheckCastPPNode(control(), ary, arytype->cast_to_not_null_free()));
+       replace_in_map(ary, better_ary);
+       ary = better_ary;
+       arytype = _gvn.type(ary)->is_aryptr();
+     }
+   }
+ 
+   if (!arytype->is_flat() && !arytype->is_not_flat()) {
+     bool flat_array = true;
+     Deoptimization::DeoptReason reason = Deoptimization::Reason_none;
+     if (arytype->speculative() != NULL &&
+         arytype->speculative()->is_aryptr()->is_not_flat() &&
+         !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
+       flat_array = false;
+       reason = Deoptimization::Reason_speculate_class_check;
+     } else if (UseArrayLoadStoreProfile && !too_many_traps_or_recompiles(reason)) {
+       ciKlass* array_type = NULL;
+       ciKlass* element_type = NULL;
+       ProfilePtrKind element_ptr = ProfileMaybeNull;
+       bool null_free_array = true;
+       method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
+       reason = Deoptimization::Reason_class_check;
+     }
+     if (!flat_array) {
+       { // Deoptimize if flat array
+         BuildCutout unless(this, flat_array_test(ary, /* flat = */ false), PROB_MAX);
+         uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
+       }
+       assert(!stopped(), "flat array should have been caught earlier");
+       Node* better_ary = _gvn.transform(new CheckCastPPNode(control(), ary, arytype->cast_to_not_flat()));
+       replace_in_map(ary, better_ary);
+       ary = better_ary;
+       arytype = _gvn.type(ary)->is_aryptr();
+     }
+   }
+ 
    // Make array address computation control dependent to prevent it
    // from floating above the range check during loop optimizations.
    Node* ptr = array_element_address(ary, idx, type, sizetype, control());
    assert(ptr != top(), "top should go hand-in-hand with stopped");
  

@@ -1418,11 +1766,11 @@
        if (C->eliminate_boxing()) {
          // Mark the successor block as parsed
          branch_block->next_path_num();
        }
      } else {                    // Path is live.
-       adjust_map_after_if(btest, c, prob, branch_block, next_block);
+       adjust_map_after_if(btest, c, prob, branch_block);
        if (!stopped()) {
          merge(target_bci);
        }
      }
    }

@@ -1436,17 +1784,16 @@
      if (C->eliminate_boxing()) {
        // Mark the successor block as parsed
        next_block->next_path_num();
      }
    } else  {                     // Path is live.
-     adjust_map_after_if(BoolTest(btest).negate(), c, 1.0-prob,
-                         next_block, branch_block);
+     adjust_map_after_if(BoolTest(btest).negate(), c, 1.0-prob, next_block);
    }
  }
  
  //------------------------------------do_if------------------------------------
- void Parse::do_if(BoolTest::mask btest, Node* c) {
+ void Parse::do_if(BoolTest::mask btest, Node* c, bool new_path, Node** ctrl_taken) {
    int target_bci = iter().get_dest();
  
    Block* branch_block = successor_for_bci(target_bci);
    Block* next_block   = successor_for_bci(iter().next_bci());
  

@@ -1526,34 +1873,427 @@
    { PreserveJVMState pjvms(this);
      taken_branch = _gvn.transform(taken_branch);
      set_control(taken_branch);
  
      if (stopped()) {
-       if (C->eliminate_boxing()) {
-         // Mark the successor block as parsed
+       if (C->eliminate_boxing() && !new_path) {
+         // Mark the successor block as parsed (if we haven't created a new path)
          branch_block->next_path_num();
        }
      } else {
-       adjust_map_after_if(taken_btest, c, prob, branch_block, next_block);
+       adjust_map_after_if(taken_btest, c, prob, branch_block);
        if (!stopped()) {
-         merge(target_bci);
+         if (new_path) {
+           // Merge by using a new path
+           merge_new_path(target_bci);
+         } else if (ctrl_taken != NULL) {
+           // Don't merge but save taken branch to be wired by caller
+           *ctrl_taken = control();
+         } else {
+           merge(target_bci);
+         }
        }
      }
    }
  
    untaken_branch = _gvn.transform(untaken_branch);
    set_control(untaken_branch);
  
    // Branch not taken.
-   if (stopped()) {
+   if (stopped() && ctrl_taken == NULL) {
      if (C->eliminate_boxing()) {
-       // Mark the successor block as parsed
+       // Mark the successor block as parsed (if caller does not re-wire control flow)
        next_block->next_path_num();
      }
    } else {
-     adjust_map_after_if(untaken_btest, c, untaken_prob,
-                         next_block, branch_block);
+     adjust_map_after_if(untaken_btest, c, untaken_prob, next_block);
+   }
+ }
+ 
+ 
+ static ProfilePtrKind speculative_ptr_kind(const TypeOopPtr* t) {
+   if (t->speculative() == NULL) {
+     return ProfileUnknownNull;
+   }
+   if (t->speculative_always_null()) {
+     return ProfileAlwaysNull;
+   }
+   if (t->speculative_maybe_null()) {
+     return ProfileMaybeNull;
+   }
+   return ProfileNeverNull;
+ }
+ 
+ void Parse::acmp_always_null_input(Node* input, const TypeOopPtr* tinput, BoolTest::mask btest, Node* eq_region) {
+   inc_sp(2);
+   Node* cast = null_check_common(input, T_OBJECT, true, NULL,
+                                  !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check) &&
+                                  speculative_ptr_kind(tinput) == ProfileAlwaysNull);
+   dec_sp(2);
+   if (btest == BoolTest::ne) {
+     {
+       PreserveJVMState pjvms(this);
+       replace_in_map(input, cast);
+       int target_bci = iter().get_dest();
+       merge(target_bci);
+     }
+     record_for_igvn(eq_region);
+     set_control(_gvn.transform(eq_region));
+   } else {
+     replace_in_map(input, cast);
+   }
+ }
+ 
+ Node* Parse::acmp_null_check(Node* input, const TypeOopPtr* tinput, ProfilePtrKind input_ptr, Node*& null_ctl) {
+   inc_sp(2);
+   null_ctl = top();
+   Node* cast = null_check_oop(input, &null_ctl,
+                               input_ptr == ProfileNeverNull || (input_ptr == ProfileUnknownNull && !too_many_traps_or_recompiles(Deoptimization::Reason_null_check)),
+                               false,
+                               speculative_ptr_kind(tinput) == ProfileNeverNull &&
+                               !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check));
+   dec_sp(2);
+   assert(!stopped(), "null input should have been caught earlier");
+   if (cast->is_InlineType()) {
+     cast = cast->as_InlineType()->get_oop();
+   }
+   return cast;
+ }
+ 
+ void Parse::acmp_known_non_inline_type_input(Node* input, const TypeOopPtr* tinput, ProfilePtrKind input_ptr, ciKlass* input_type, BoolTest::mask btest, Node* eq_region) {
+   Node* ne_region = new RegionNode(1);
+   Node* null_ctl;
+   Node* cast = acmp_null_check(input, tinput, input_ptr, null_ctl);
+   ne_region->add_req(null_ctl);
+ 
+   Node* slow_ctl = type_check_receiver(cast, input_type, 1.0, &cast);
+   {
+     PreserveJVMState pjvms(this);
+     inc_sp(2);
+     set_control(slow_ctl);
+     Deoptimization::DeoptReason reason;
+     if (tinput->speculative_type() != NULL && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
+       reason = Deoptimization::Reason_speculate_class_check;
+     } else {
+       reason = Deoptimization::Reason_class_check;
+     }
+     uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
+   }
+   ne_region->add_req(control());
+ 
+   record_for_igvn(ne_region);
+   set_control(_gvn.transform(ne_region));
+   if (btest == BoolTest::ne) {
+     {
+       PreserveJVMState pjvms(this);
+       if (null_ctl == top()) {
+         replace_in_map(input, cast);
+       }
+       int target_bci = iter().get_dest();
+       merge(target_bci);
+     }
+     record_for_igvn(eq_region);
+     set_control(_gvn.transform(eq_region));
+   } else {
+     if (null_ctl == top()) {
+       replace_in_map(input, cast);
+     }
+     set_control(_gvn.transform(ne_region));
+   }
+ }
+ 
+ void Parse::acmp_unknown_non_inline_type_input(Node* input, const TypeOopPtr* tinput, ProfilePtrKind input_ptr, BoolTest::mask btest, Node* eq_region) {
+   Node* ne_region = new RegionNode(1);
+   Node* null_ctl;
+   Node* cast = acmp_null_check(input, tinput, input_ptr, null_ctl);
+   ne_region->add_req(null_ctl);
+ 
+   {
+     BuildCutout unless(this, inline_type_test(cast, /* is_inline = */ false), PROB_MAX);
+     inc_sp(2);
+     uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile);
+   }
+ 
+   ne_region->add_req(control());
+ 
+   record_for_igvn(ne_region);
+   set_control(_gvn.transform(ne_region));
+   if (btest == BoolTest::ne) {
+     {
+       PreserveJVMState pjvms(this);
+       if (null_ctl == top()) {
+         replace_in_map(input, cast);
+       }
+       int target_bci = iter().get_dest();
+       merge(target_bci);
+     }
+     record_for_igvn(eq_region);
+     set_control(_gvn.transform(eq_region));
+   } else {
+     if (null_ctl == top()) {
+       replace_in_map(input, cast);
+     }
+     set_control(_gvn.transform(ne_region));
+   }
+ }
+ 
+ void Parse::do_acmp(BoolTest::mask btest, Node* left, Node* right) {
+   ciKlass* left_type = NULL;
+   ciKlass* right_type = NULL;
+   ProfilePtrKind left_ptr = ProfileUnknownNull;
+   ProfilePtrKind right_ptr = ProfileUnknownNull;
+   bool left_inline_type = true;
+   bool right_inline_type = true;
+ 
+   // Leverage profiling at acmp
+   if (UseACmpProfile) {
+     method()->acmp_profiled_type(bci(), left_type, right_type, left_ptr, right_ptr, left_inline_type, right_inline_type);
+     if (too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) {
+       left_type = NULL;
+       right_type = NULL;
+       left_inline_type = true;
+       right_inline_type = true;
+     }
+     if (too_many_traps_or_recompiles(Deoptimization::Reason_null_check)) {
+       left_ptr = ProfileUnknownNull;
+       right_ptr = ProfileUnknownNull;
+     }
+   }
+ 
+   if (UseTypeSpeculation) {
+     record_profile_for_speculation(left, left_type, left_ptr);
+     record_profile_for_speculation(right, right_type, right_ptr);
+   }
+ 
+   if (!EnableValhalla) {
+     Node* cmp = CmpP(left, right);
+     cmp = optimize_cmp_with_klass(cmp);
+     do_if(btest, cmp);
+     return;
+   }
+ 
+   // Check for equality before potentially allocating
+   if (left == right) {
+     do_if(btest, makecon(TypeInt::CC_EQ));
+     return;
+   }
+ 
+   // Allocate inline type operands and re-execute on deoptimization
+   if (left->is_InlineType()) {
+     PreserveReexecuteState preexecs(this);
+     inc_sp(2);
+     jvms()->set_should_reexecute(true);
+     left = left->as_InlineType()->buffer(this)->get_oop();
+   }
+   if (right->is_InlineType()) {
+     PreserveReexecuteState preexecs(this);
+     inc_sp(2);
+     jvms()->set_should_reexecute(true);
+     right = right->as_InlineType()->buffer(this)->get_oop();
+   }
+ 
+   // First, do a normal pointer comparison
+   const TypeOopPtr* tleft = _gvn.type(left)->isa_oopptr();
+   const TypeOopPtr* tright = _gvn.type(right)->isa_oopptr();
+   Node* cmp = CmpP(left, right);
+   cmp = optimize_cmp_with_klass(cmp);
+   if (tleft == NULL || !tleft->can_be_inline_type() ||
+       tright == NULL || !tright->can_be_inline_type()) {
+     // This is sufficient, if one of the operands can't be an inline type
+     do_if(btest, cmp);
+     return;
+   }
+   Node* eq_region = NULL;
+   if (btest == BoolTest::eq) {
+     do_if(btest, cmp, true);
+     if (stopped()) {
+       return;
+     }
+   } else {
+     assert(btest == BoolTest::ne, "only eq or ne");
+     Node* is_not_equal = NULL;
+     eq_region = new RegionNode(3);
+     {
+       PreserveJVMState pjvms(this);
+       do_if(btest, cmp, false, &is_not_equal);
+       if (!stopped()) {
+         eq_region->init_req(1, control());
+       }
+     }
+     if (is_not_equal == NULL || is_not_equal->is_top()) {
+       record_for_igvn(eq_region);
+       set_control(_gvn.transform(eq_region));
+       return;
+     }
+     set_control(is_not_equal);
+   }
+ 
+   // Prefer speculative types if available
+   if (!too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
+     if (tleft->speculative_type() != NULL) {
+       left_type = tleft->speculative_type();
+     }
+     if (tright->speculative_type() != NULL) {
+       right_type = tright->speculative_type();
+     }
+   }
+ 
+   if (speculative_ptr_kind(tleft) != ProfileMaybeNull && speculative_ptr_kind(tleft) != ProfileUnknownNull) {
+     ProfilePtrKind speculative_left_ptr = speculative_ptr_kind(tleft);
+     if (speculative_left_ptr == ProfileAlwaysNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_assert)) {
+       left_ptr = speculative_left_ptr;
+     } else if (speculative_left_ptr == ProfileNeverNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check)) {
+       left_ptr = speculative_left_ptr;
+     }
+   }
+   if (speculative_ptr_kind(tright) != ProfileMaybeNull && speculative_ptr_kind(tright) != ProfileUnknownNull) {
+     ProfilePtrKind speculative_right_ptr = speculative_ptr_kind(tright);
+     if (speculative_right_ptr == ProfileAlwaysNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_assert)) {
+       right_ptr = speculative_right_ptr;
+     } else if (speculative_right_ptr == ProfileNeverNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check)) {
+       right_ptr = speculative_right_ptr;
+     }
+   }
+ 
+   if (left_ptr == ProfileAlwaysNull) {
+     // Comparison with null. Assert the input is indeed null and we're done.
+     acmp_always_null_input(left, tleft, btest, eq_region);
+     return;
+   }
+   if (right_ptr == ProfileAlwaysNull) {
+     // Comparison with null. Assert the input is indeed null and we're done.
+     acmp_always_null_input(right, tright, btest, eq_region);
+     return;
+   }
+   if (left_type != NULL && !left_type->is_inlinetype()) {
+     // Comparison with an object of known type
+     acmp_known_non_inline_type_input(left, tleft, left_ptr, left_type, btest, eq_region);
+     return;
+   }
+   if (right_type != NULL && !right_type->is_inlinetype()) {
+     // Comparison with an object of known type
+     acmp_known_non_inline_type_input(right, tright, right_ptr, right_type, btest, eq_region);
+     return;
+   }
+   if (!left_inline_type) {
+     // Comparison with an object known not to be an inline type
+     acmp_unknown_non_inline_type_input(left, tleft, left_ptr, btest, eq_region);
+     return;
+   }
+   if (!right_inline_type) {
+     // Comparison with an object known not to be an inline type
+     acmp_unknown_non_inline_type_input(right, tright, right_ptr, btest, eq_region);
+     return;
+   }
+ 
+   // Pointers are not equal, check if first operand is non-null
+   Node* ne_region = new RegionNode(6);
+   Node* null_ctl;
+   Node* not_null_right = acmp_null_check(right, tright, right_ptr, null_ctl);
+   ne_region->init_req(1, null_ctl);
+ 
+   // First operand is non-null, check if it is an inline type
+   Node* is_value = inline_type_test(not_null_right);
+   IfNode* is_value_iff = create_and_map_if(control(), is_value, PROB_FAIR, COUNT_UNKNOWN);
+   Node* not_value = _gvn.transform(new IfFalseNode(is_value_iff));
+   ne_region->init_req(2, not_value);
+   set_control(_gvn.transform(new IfTrueNode(is_value_iff)));
+ 
+   // The first operand is an inline type, check if the second operand is non-null
+   Node* not_null_left = acmp_null_check(left, tleft, left_ptr, null_ctl);
+   ne_region->init_req(3, null_ctl);
+ 
+   // Check if both operands are of the same class.
+   Node* kls_left = load_object_klass(not_null_left);
+   Node* kls_right = load_object_klass(not_null_right);
+   Node* kls_cmp = CmpP(kls_left, kls_right);
+   Node* kls_bol = _gvn.transform(new BoolNode(kls_cmp, BoolTest::ne));
+   IfNode* kls_iff = create_and_map_if(control(), kls_bol, PROB_FAIR, COUNT_UNKNOWN);
+   Node* kls_ne = _gvn.transform(new IfTrueNode(kls_iff));
+   set_control(_gvn.transform(new IfFalseNode(kls_iff)));
+   ne_region->init_req(4, kls_ne);
+ 
+   if (stopped()) {
+     record_for_igvn(ne_region);
+     set_control(_gvn.transform(ne_region));
+     if (btest == BoolTest::ne) {
+       {
+         PreserveJVMState pjvms(this);
+         int target_bci = iter().get_dest();
+         merge(target_bci);
+       }
+       record_for_igvn(eq_region);
+       set_control(_gvn.transform(eq_region));
+     }
+     return;
+   }
+ 
+   // Both operands are values types of the same class, we need to perform a
+   // substitutability test. Delegate to PrimitiveObjectMethods::isSubstitutable().
+   Node* ne_io_phi = PhiNode::make(ne_region, i_o());
+   Node* mem = reset_memory();
+   Node* ne_mem_phi = PhiNode::make(ne_region, mem);
+ 
+   Node* eq_io_phi = NULL;
+   Node* eq_mem_phi = NULL;
+   if (eq_region != NULL) {
+     eq_io_phi = PhiNode::make(eq_region, i_o());
+     eq_mem_phi = PhiNode::make(eq_region, mem);
+   }
+ 
+   set_all_memory(mem);
+ 
+   kill_dead_locals();
+   ciMethod* subst_method = ciEnv::current()->PrimitiveObjectMethods_klass()->find_method(ciSymbols::isSubstitutable_name(), ciSymbols::object_object_boolean_signature());
+   CallStaticJavaNode *call = new CallStaticJavaNode(C, TypeFunc::make(subst_method), SharedRuntime::get_resolve_static_call_stub(), subst_method);
+   call->set_override_symbolic_info(true);
+   call->init_req(TypeFunc::Parms, not_null_left);
+   call->init_req(TypeFunc::Parms+1, not_null_right);
+   inc_sp(2);
+   set_edges_for_java_call(call, false, false);
+   Node* ret = set_results_for_java_call(call, false, true);
+   dec_sp(2);
+ 
+   // Test the return value of PrimitiveObjectMethods::isSubstitutable()
+   Node* subst_cmp = _gvn.transform(new CmpINode(ret, intcon(1)));
+   Node* ctl = C->top();
+   if (btest == BoolTest::eq) {
+     PreserveJVMState pjvms(this);
+     do_if(btest, subst_cmp);
+     if (!stopped()) {
+       ctl = control();
+     }
+   } else {
+     assert(btest == BoolTest::ne, "only eq or ne");
+     PreserveJVMState pjvms(this);
+     do_if(btest, subst_cmp, false, &ctl);
+     if (!stopped()) {
+       eq_region->init_req(2, control());
+       eq_io_phi->init_req(2, i_o());
+       eq_mem_phi->init_req(2, reset_memory());
+     }
+   }
+   ne_region->init_req(5, ctl);
+   ne_io_phi->init_req(5, i_o());
+   ne_mem_phi->init_req(5, reset_memory());
+ 
+   record_for_igvn(ne_region);
+   set_control(_gvn.transform(ne_region));
+   set_i_o(_gvn.transform(ne_io_phi));
+   set_all_memory(_gvn.transform(ne_mem_phi));
+ 
+   if (btest == BoolTest::ne) {
+     {
+       PreserveJVMState pjvms(this);
+       int target_bci = iter().get_dest();
+       merge(target_bci);
+     }
+ 
+     record_for_igvn(eq_region);
+     set_control(_gvn.transform(eq_region));
+     set_i_o(_gvn.transform(eq_io_phi));
+     set_all_memory(_gvn.transform(eq_mem_phi));
    }
  }
  
  bool Parse::path_is_suitable_for_uncommon_trap(float prob) const {
    // Don't want to speculate on uncommon traps when running with -Xcomp

@@ -1579,12 +2319,11 @@
  // Adjust the JVM state to reflect the result of taking this path.
  // Basically, it means inspecting the CmpNode controlling this
  // branch, seeing how it constrains a tested value, and then
  // deciding if it's worth our while to encode this constraint
  // as graph nodes in the current abstract interpretation map.
- void Parse::adjust_map_after_if(BoolTest::mask btest, Node* c, float prob,
-                                 Block* path, Block* other_path) {
+ void Parse::adjust_map_after_if(BoolTest::mask btest, Node* c, float prob, Block* path) {
    if (!c->is_Cmp()) {
      maybe_add_predicate_after_if(path);
      return;
    }
  

@@ -1790,10 +2529,14 @@
        if (obj_type->speculative_type_not_null() != NULL) {
          ciKlass* k = obj_type->speculative_type();
          inc_sp(2);
          obj = maybe_cast_profiled_obj(obj, k);
          dec_sp(2);
+         if (obj->is_InlineType()) {
+           assert(obj->as_InlineType()->is_allocated(&_gvn), "must be allocated");
+           obj = obj->as_InlineType()->get_oop();
+         }
          // Make the CmpP use the casted obj
          addp = basic_plus_adr(obj, addp->in(AddPNode::Offset));
          load_klass = load_klass->clone();
          load_klass->set_req(2, addp);
          load_klass = _gvn.transform(load_klass);

@@ -2637,37 +3380,40 @@
    handle_if_null:
      // If this is a backwards branch in the bytecodes, add Safepoint
      maybe_add_safepoint(iter().get_dest());
      a = null();
      b = pop();
-     if (!_gvn.type(b)->speculative_maybe_null() &&
-         !too_many_traps(Deoptimization::Reason_speculate_null_check)) {
-       inc_sp(1);
-       Node* null_ctl = top();
-       b = null_check_oop(b, &null_ctl, true, true, true);
-       assert(null_ctl->is_top(), "no null control here");
-       dec_sp(1);
-     } else if (_gvn.type(b)->speculative_always_null() &&
-                !too_many_traps(Deoptimization::Reason_speculate_null_assert)) {
-       inc_sp(1);
-       b = null_assert(b);
-       dec_sp(1);
-     }
-     c = _gvn.transform( new CmpPNode(b, a) );
+     if (b->is_InlineType()) {
+       // Return constant false because 'b' is always non-null
+       c = _gvn.makecon(TypeInt::CC_GT);
+     } else {
+       if (!_gvn.type(b)->speculative_maybe_null() &&
+           !too_many_traps(Deoptimization::Reason_speculate_null_check)) {
+         inc_sp(1);
+         Node* null_ctl = top();
+         b = null_check_oop(b, &null_ctl, true, true, true);
+         assert(null_ctl->is_top(), "no null control here");
+         dec_sp(1);
+       } else if (_gvn.type(b)->speculative_always_null() &&
+                  !too_many_traps(Deoptimization::Reason_speculate_null_assert)) {
+         inc_sp(1);
+         b = null_assert(b);
+         dec_sp(1);
+       }
+       c = _gvn.transform( new CmpPNode(b, a) );
+     }
      do_ifnull(btest, c);
      break;
  
    case Bytecodes::_if_acmpeq: btest = BoolTest::eq; goto handle_if_acmp;
    case Bytecodes::_if_acmpne: btest = BoolTest::ne; goto handle_if_acmp;
    handle_if_acmp:
      // If this is a backwards branch in the bytecodes, add Safepoint
      maybe_add_safepoint(iter().get_dest());
      a = pop();
      b = pop();
-     c = _gvn.transform( new CmpPNode(b, a) );
-     c = optimize_cmp_with_klass(c);
-     do_if(btest, c);
+     do_acmp(btest, b, a);
      break;
  
    case Bytecodes::_ifeq: btest = BoolTest::eq; goto handle_ifxx;
    case Bytecodes::_ifne: btest = BoolTest::ne; goto handle_ifxx;
    case Bytecodes::_iflt: btest = BoolTest::lt; goto handle_ifxx;

@@ -2718,21 +3464,27 @@
      break;
    case Bytecodes::_instanceof:
      do_instanceof();
      break;
    case Bytecodes::_anewarray:
-     do_anewarray();
+     do_newarray();
      break;
    case Bytecodes::_newarray:
      do_newarray((BasicType)iter().get_index());
      break;
    case Bytecodes::_multianewarray:
      do_multianewarray();
      break;
    case Bytecodes::_new:
      do_new();
      break;
+   case Bytecodes::_defaultvalue:
+     do_defaultvalue();
+     break;
+   case Bytecodes::_withfield:
+     do_withfield();
+     break;
  
    case Bytecodes::_jsr:
    case Bytecodes::_jsr_w:
      do_jsr();
      break;
< prev index next >