< prev index next >

src/hotspot/share/opto/macro.cpp

Print this page
*** 21,10 ***
--- 21,11 ---
   * questions.
   *
   */
  
  #include "precompiled.hpp"
+ #include "ci/ciFlatArrayKlass.hpp"
  #include "compiler/compileLog.hpp"
  #include "gc/shared/collectedHeap.inline.hpp"
  #include "gc/shared/tlab_globals.hpp"
  #include "libadt/vectset.hpp"
  #include "memory/universe.hpp"

*** 34,10 ***
--- 35,11 ---
  #include "opto/castnode.hpp"
  #include "opto/cfgnode.hpp"
  #include "opto/compile.hpp"
  #include "opto/convertnode.hpp"
  #include "opto/graphKit.hpp"
+ #include "opto/inlinetypenode.hpp"
  #include "opto/intrinsicnode.hpp"
  #include "opto/locknode.hpp"
  #include "opto/loopnode.hpp"
  #include "opto/macro.hpp"
  #include "opto/memnode.hpp"

*** 50,10 ***
--- 52,11 ---
  #include "opto/subnode.hpp"
  #include "opto/subtypenode.hpp"
  #include "opto/type.hpp"
  #include "prims/jvmtiExport.hpp"
  #include "runtime/sharedRuntime.hpp"
+ #include "runtime/stubRoutines.hpp"
  #include "utilities/macros.hpp"
  #include "utilities/powerOfTwo.hpp"
  #if INCLUDE_G1GC
  #include "gc/g1/g1ThreadLocalData.hpp"
  #endif // INCLUDE_G1GC

*** 82,22 ***
      }
    }
    return nreplacements;
  }
  
- void PhaseMacroExpand::migrate_outs(Node *old, Node *target) {
-   assert(old != NULL, "sanity");
-   for (DUIterator_Fast imax, i = old->fast_outs(imax); i < imax; i++) {
-     Node* use = old->fast_out(i);
-     _igvn.rehash_node_delayed(use);
-     imax -= replace_input(use, old, target);
-     // back up iterator
-     --i;
-   }
-   assert(old->outcnt() == 0, "all uses must be deleted");
- }
- 
  Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) {
    Node* cmp;
    if (mask != 0) {
      Node* and_node = transform_later(new AndXNode(word, MakeConX(mask)));
      cmp = transform_later(new CmpXNode(and_node, MakeConX(bits)));
--- 85,10 ---

*** 156,11 ***
    return call;
  }
  
  void PhaseMacroExpand::eliminate_gc_barrier(Node* p2x) {
    BarrierSetC2 *bs = BarrierSet::barrier_set()->barrier_set_c2();
!   bs->eliminate_gc_barrier(this, p2x);
  }
  
  // Search for a memory operation for the specified memory slice.
  static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_mem, Node *alloc, PhaseGVN *phase) {
    Node *orig_mem = mem;
--- 147,11 ---
    return call;
  }
  
  void PhaseMacroExpand::eliminate_gc_barrier(Node* p2x) {
    BarrierSetC2 *bs = BarrierSet::barrier_set()->barrier_set_c2();
!   bs->eliminate_gc_barrier(&_igvn, p2x);
  }
  
  // Search for a memory operation for the specified memory slice.
  static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_mem, Node *alloc, PhaseGVN *phase) {
    Node *orig_mem = mem;

*** 205,11 ***
      } else if (mem->is_Store()) {
        const TypePtr* atype = mem->as_Store()->adr_type();
        int adr_idx = phase->C->get_alias_index(atype);
        if (adr_idx == alias_idx) {
          assert(atype->isa_oopptr(), "address type must be oopptr");
!         int adr_offset = atype->offset();
          uint adr_iid = atype->is_oopptr()->instance_id();
          // Array elements references have the same alias_idx
          // but different offset and different instance_id.
          if (adr_offset == offset && adr_iid == alloc->_idx) {
            return mem;
--- 196,11 ---
      } else if (mem->is_Store()) {
        const TypePtr* atype = mem->as_Store()->adr_type();
        int adr_idx = phase->C->get_alias_index(atype);
        if (adr_idx == alias_idx) {
          assert(atype->isa_oopptr(), "address type must be oopptr");
!         int adr_offset = atype->flattened_offset();
          uint adr_iid = atype->is_oopptr()->instance_id();
          // Array elements references have the same alias_idx
          // but different offset and different instance_id.
          if (adr_offset == offset && adr_iid == alloc->_idx) {
            return mem;

*** 250,11 ***
          DEBUG_ONLY(mem->dump();)
          assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field");
          return NULL;
        }
        mem = mem->in(MemNode::Memory);
!    } else if (mem->Opcode() == Op_StrInflatedCopy) {
        Node* adr = mem->in(3); // Destination array
        const TypePtr* atype = adr->bottom_type()->is_ptr();
        int adr_idx = phase->C->get_alias_index(atype);
        if (adr_idx == alias_idx) {
          DEBUG_ONLY(mem->dump();)
--- 241,11 ---
          DEBUG_ONLY(mem->dump();)
          assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field");
          return NULL;
        }
        mem = mem->in(MemNode::Memory);
!     } else if (mem->Opcode() == Op_StrInflatedCopy) {
        Node* adr = mem->in(3); // Destination array
        const TypePtr* atype = adr->bottom_type()->is_ptr();
        int adr_idx = phase->C->get_alias_index(atype);
        if (adr_idx == alias_idx) {
          DEBUG_ONLY(mem->dump();)

*** 295,45 ***
        Node* dest_pos = ac->in(ArrayCopyNode::DestPos);
        const TypeInt* src_pos_t = _igvn.type(src_pos)->is_int();
        const TypeInt* dest_pos_t = _igvn.type(dest_pos)->is_int();
  
        Node* adr = NULL;
!       const TypePtr* adr_type = NULL;
        if (src_pos_t->is_con() && dest_pos_t->is_con()) {
          intptr_t off = ((src_pos_t->get_con() - dest_pos_t->get_con()) << shift) + offset;
-         Node* base = ac->in(ArrayCopyNode::Src);
          adr = _igvn.transform(new AddPNode(base, base, MakeConX(off)));
!         adr_type = _igvn.type(base)->is_ptr()->add_offset(off);
          if (ac->in(ArrayCopyNode::Src) == ac->in(ArrayCopyNode::Dest)) {
            // Don't emit a new load from src if src == dst but try to get the value from memory instead
            return value_from_mem(ac->in(TypeFunc::Memory), ctl, ft, ftype, adr_type->isa_oopptr(), alloc);
          }
        } else {
          Node* diff = _igvn.transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
  #ifdef _LP64
          diff = _igvn.transform(new ConvI2LNode(diff));
  #endif
          diff = _igvn.transform(new LShiftXNode(diff, intcon(shift)));
  
          Node* off = _igvn.transform(new AddXNode(MakeConX(offset), diff));
-         Node* base = ac->in(ArrayCopyNode::Src);
          adr = _igvn.transform(new AddPNode(base, base, off));
!         adr_type = _igvn.type(base)->is_ptr()->add_offset(Type::OffsetBot);
!         if (ac->in(ArrayCopyNode::Src) == ac->in(ArrayCopyNode::Dest)) {
!           // Non constant offset in the array: we can't statically
!           // determine the value
!           return NULL;
-         }
        }
        MergeMemNode* mergemen = _igvn.transform(MergeMemNode::make(mem))->as_MergeMem();
        BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
        res = ArrayCopyNode::load(bs, &_igvn, ctl, mergemen, adr, adr_type, type, bt);
      }
    }
    if (res != NULL) {
      if (ftype->isa_narrowoop()) {
        // PhaseMacroExpand::scalar_replacement adds DecodeN nodes
        res = _igvn.transform(new EncodePNode(res, ftype));
      }
      return res;
    }
    return NULL;
--- 286,55 ---
        Node* dest_pos = ac->in(ArrayCopyNode::DestPos);
        const TypeInt* src_pos_t = _igvn.type(src_pos)->is_int();
        const TypeInt* dest_pos_t = _igvn.type(dest_pos)->is_int();
  
        Node* adr = NULL;
!       Node* base = ac->in(ArrayCopyNode::Src);
+       const TypePtr* adr_type = _igvn.type(base)->is_ptr();
+       assert(adr_type->isa_aryptr(), "only arrays here");
+       if (adr_type->is_aryptr()->is_flat()) {
+         ciFlatArrayKlass* vak = adr_type->is_aryptr()->klass()->as_flat_array_klass();
+         shift = vak->log2_element_size();
+       }
        if (src_pos_t->is_con() && dest_pos_t->is_con()) {
          intptr_t off = ((src_pos_t->get_con() - dest_pos_t->get_con()) << shift) + offset;
          adr = _igvn.transform(new AddPNode(base, base, MakeConX(off)));
!         adr_type = _igvn.type(adr)->is_ptr();
+         assert(adr_type == _igvn.type(base)->is_aryptr()->add_field_offset_and_offset(off), "incorrect address type");
          if (ac->in(ArrayCopyNode::Src) == ac->in(ArrayCopyNode::Dest)) {
            // Don't emit a new load from src if src == dst but try to get the value from memory instead
            return value_from_mem(ac->in(TypeFunc::Memory), ctl, ft, ftype, adr_type->isa_oopptr(), alloc);
          }
        } else {
+         if (ac->in(ArrayCopyNode::Src) == ac->in(ArrayCopyNode::Dest)) {
+           // Non constant offset in the array: we can't statically
+           // determine the value
+           return NULL;
+         }
          Node* diff = _igvn.transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
  #ifdef _LP64
          diff = _igvn.transform(new ConvI2LNode(diff));
  #endif
          diff = _igvn.transform(new LShiftXNode(diff, intcon(shift)));
  
          Node* off = _igvn.transform(new AddXNode(MakeConX(offset), diff));
          adr = _igvn.transform(new AddPNode(base, base, off));
!         // In the case of a flattened inline type array, each field has its
!         // own slice so we need to extract the field being accessed from
!         // the address computation
!         adr_type = adr_type->is_aryptr()->add_field_offset_and_offset(offset)->add_offset(Type::OffsetBot);
!         adr = _igvn.transform(new CastPPNode(adr, adr_type));
        }
        MergeMemNode* mergemen = _igvn.transform(MergeMemNode::make(mem))->as_MergeMem();
        BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
        res = ArrayCopyNode::load(bs, &_igvn, ctl, mergemen, adr, adr_type, type, bt);
      }
    }
    if (res != NULL) {
      if (ftype->isa_narrowoop()) {
        // PhaseMacroExpand::scalar_replacement adds DecodeN nodes
+       assert(res->isa_DecodeN(), "should be narrow oop");
        res = _igvn.transform(new EncodePNode(res, ftype));
      }
      return res;
    }
    return NULL;

*** 345,11 ***
  // Note: this function is recursive, its depth is limited by the "level" argument
  // Returns the computed Phi, or NULL if it cannot compute it.
  Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level) {
    assert(mem->is_Phi(), "sanity");
    int alias_idx = C->get_alias_index(adr_t);
!   int offset = adr_t->offset();
    int instance_id = adr_t->instance_id();
  
    // Check if an appropriate value phi already exists.
    Node* region = mem->in(0);
    for (DUIterator_Fast kmax, k = region->fast_outs(kmax); k < kmax; k++) {
--- 346,11 ---
  // Note: this function is recursive, its depth is limited by the "level" argument
  // Returns the computed Phi, or NULL if it cannot compute it.
  Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level) {
    assert(mem->is_Phi(), "sanity");
    int alias_idx = C->get_alias_index(adr_t);
!   int offset = adr_t->flattened_offset();
    int instance_id = adr_t->instance_id();
  
    // Check if an appropriate value phi already exists.
    Node* region = mem->in(0);
    for (DUIterator_Fast kmax, k = region->fast_outs(kmax); k < kmax; k++) {

*** 384,11 ***
        values.at_put(j, in);
      } else  {
        Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
        if (val == start_mem || val == alloc_mem) {
          // hit a sentinel, return appropriate 0 value
!         values.at_put(j, _igvn.zerocon(ft));
          continue;
        }
        if (val->is_Initialize()) {
          val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
        }
--- 385,17 ---
        values.at_put(j, in);
      } else  {
        Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
        if (val == start_mem || val == alloc_mem) {
          // hit a sentinel, return appropriate 0 value
!         Node* default_value = alloc->in(AllocateNode::DefaultValue);
+         if (default_value != NULL) {
+           values.at_put(j, default_value);
+         } else {
+           assert(alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
+           values.at_put(j, _igvn.zerocon(ft));
+         }
          continue;
        }
        if (val->is_Initialize()) {
          val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
        }

*** 404,11 ***
          if (is_subword_type(ft)) {
            n = Compile::narrow_value(ft, n, phi_type, &_igvn, true);
          }
          values.at_put(j, n);
        } else if(val->is_Proj() && val->in(0) == alloc) {
!         values.at_put(j, _igvn.zerocon(ft));
        } else if (val->is_Phi()) {
          val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1);
          if (val == NULL) {
            return NULL;
          }
--- 411,17 ---
          if (is_subword_type(ft)) {
            n = Compile::narrow_value(ft, n, phi_type, &_igvn, true);
          }
          values.at_put(j, n);
        } else if(val->is_Proj() && val->in(0) == alloc) {
!         Node* default_value = alloc->in(AllocateNode::DefaultValue);
+         if (default_value != NULL) {
+           values.at_put(j, default_value);
+         } else {
+           assert(alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
+           values.at_put(j, _igvn.zerocon(ft));
+         }
        } else if (val->is_Phi()) {
          val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1);
          if (val == NULL) {
            return NULL;
          }

*** 448,13 ***
    assert(adr_t->is_known_instance_field(), "instance required");
    int instance_id = adr_t->instance_id();
    assert((uint)instance_id == alloc->_idx, "wrong allocation");
  
    int alias_idx = C->get_alias_index(adr_t);
!   int offset = adr_t->offset();
    Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory);
-   Node *alloc_ctrl = alloc->in(TypeFunc::Control);
    Node *alloc_mem = alloc->in(TypeFunc::Memory);
    VectorSet visited;
  
    bool done = sfpt_mem == alloc_mem;
    Node *mem = sfpt_mem;
--- 461,12 ---
    assert(adr_t->is_known_instance_field(), "instance required");
    int instance_id = adr_t->instance_id();
    assert((uint)instance_id == alloc->_idx, "wrong allocation");
  
    int alias_idx = C->get_alias_index(adr_t);
!   int offset = adr_t->flattened_offset();
    Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory);
    Node *alloc_mem = alloc->in(TypeFunc::Memory);
    VectorSet visited;
  
    bool done = sfpt_mem == alloc_mem;
    Node *mem = sfpt_mem;

*** 466,21 ***
      if (mem == start_mem || mem == alloc_mem) {
        done = true;  // hit a sentinel, return appropriate 0 value
      } else if (mem->is_Initialize()) {
        mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
        if (mem == NULL) {
!         done = true; // Something go wrong.
        } else if (mem->is_Store()) {
          const TypePtr* atype = mem->as_Store()->adr_type();
          assert(C->get_alias_index(atype) == Compile::AliasIdxRaw, "store is correct memory slice");
          done = true;
        }
      } else if (mem->is_Store()) {
        const TypeOopPtr* atype = mem->as_Store()->adr_type()->isa_oopptr();
        assert(atype != NULL, "address type must be oopptr");
        assert(C->get_alias_index(atype) == alias_idx &&
!              atype->is_known_instance_field() && atype->offset() == offset &&
               atype->instance_id() == instance_id, "store is correct memory slice");
        done = true;
      } else if (mem->is_Phi()) {
        // try to find a phi's unique input
        Node *unique_input = NULL;
--- 478,21 ---
      if (mem == start_mem || mem == alloc_mem) {
        done = true;  // hit a sentinel, return appropriate 0 value
      } else if (mem->is_Initialize()) {
        mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
        if (mem == NULL) {
!         done = true; // Something went wrong.
        } else if (mem->is_Store()) {
          const TypePtr* atype = mem->as_Store()->adr_type();
          assert(C->get_alias_index(atype) == Compile::AliasIdxRaw, "store is correct memory slice");
          done = true;
        }
      } else if (mem->is_Store()) {
        const TypeOopPtr* atype = mem->as_Store()->adr_type()->isa_oopptr();
        assert(atype != NULL, "address type must be oopptr");
        assert(C->get_alias_index(atype) == alias_idx &&
!              atype->is_known_instance_field() && atype->flattened_offset() == offset &&
               atype->instance_id() == instance_id, "store is correct memory slice");
        done = true;
      } else if (mem->is_Phi()) {
        // try to find a phi's unique input
        Node *unique_input = NULL;

*** 509,10 ***
--- 521,15 ---
      }
    }
    if (mem != NULL) {
      if (mem == start_mem || mem == alloc_mem) {
        // hit a sentinel, return appropriate 0 value
+       Node* default_value = alloc->in(AllocateNode::DefaultValue);
+       if (default_value != NULL) {
+         return default_value;
+       }
+       assert(alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
        return _igvn.zerocon(ft);
      } else if (mem->is_Store()) {
        Node* n = mem->in(MemNode::ValueIn);
        BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
        n = bs->step_over_gc_barrier(n);

*** 540,30 ***
          m = sfpt_mem;
        }
        return make_arraycopy_load(mem->as_ArrayCopy(), offset, ctl, m, ft, ftype, alloc);
      }
    }
!   // Something go wrong.
    return NULL;
  }
  
  // Check the possibility of scalar replacement.
  bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) {
    //  Scan the uses of the allocation to check for anything that would
    //  prevent us from eliminating it.
    NOT_PRODUCT( const char* fail_eliminate = NULL; )
    DEBUG_ONLY( Node* disq_node = NULL; )
    bool  can_eliminate = true;
  
    Node* res = alloc->result_cast();
    const TypeOopPtr* res_type = NULL;
    if (res == NULL) {
      // All users were eliminated.
    } else if (!res->is_CheckCastPP()) {
      NOT_PRODUCT(fail_eliminate = "Allocation does not have unique CheckCastPP";)
      can_eliminate = false;
    } else {
      res_type = _igvn.type(res)->isa_oopptr();
      if (res_type == NULL) {
        NOT_PRODUCT(fail_eliminate = "Neither instance or array allocation";)
        can_eliminate = false;
      } else if (res_type->isa_aryptr()) {
--- 557,74 ---
          m = sfpt_mem;
        }
        return make_arraycopy_load(mem->as_ArrayCopy(), offset, ctl, m, ft, ftype, alloc);
      }
    }
!   // Something went wrong.
    return NULL;
  }
  
+ // Search the last value stored into the inline type's fields.
+ Node* PhaseMacroExpand::inline_type_from_mem(Node* mem, Node* ctl, ciInlineKlass* vk, const TypeAryPtr* adr_type, int offset, AllocateNode* alloc) {
+   // Subtract the offset of the first field to account for the missing oop header
+   offset -= vk->first_field_offset();
+   // Create a new InlineTypeNode and retrieve the field values from memory
+   InlineTypeNode* vt = InlineTypeNode::make_uninitialized(_igvn, vk)->as_InlineType();
+   transform_later(vt);
+   for (int i = 0; i < vk->nof_declared_nonstatic_fields(); ++i) {
+     ciType* field_type = vt->field_type(i);
+     int field_offset = offset + vt->field_offset(i);
+     Node* value = NULL;
+     if (vt->field_is_flattened(i)) {
+       value = inline_type_from_mem(mem, ctl, field_type->as_inline_klass(), adr_type, field_offset, alloc);
+     } else {
+       const Type* ft = Type::get_const_type(field_type);
+       BasicType bt = field_type->basic_type();
+       if (UseCompressedOops && !is_java_primitive(bt)) {
+         ft = ft->make_narrowoop();
+         bt = T_NARROWOOP;
+       }
+       // Each inline type field has its own memory slice
+       adr_type = adr_type->with_field_offset(field_offset);
+       value = value_from_mem(mem, ctl, bt, ft, adr_type, alloc);
+       if (value != NULL && ft->isa_narrowoop()) {
+         assert(UseCompressedOops, "unexpected narrow oop");
+         if (value->is_EncodeP()) {
+           value = value->in(1);
+         } else {
+           value = transform_later(new DecodeNNode(value, value->get_ptr_type()));
+         }
+       }
+     }
+     if (value != NULL) {
+       vt->set_field_value(i, value);
+     } else {
+       // We might have reached the TrackedInitializationLimit
+       return NULL;
+     }
+   }
+   return vt;
+ }
+ 
  // Check the possibility of scalar replacement.
  bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) {
    //  Scan the uses of the allocation to check for anything that would
    //  prevent us from eliminating it.
    NOT_PRODUCT( const char* fail_eliminate = NULL; )
    DEBUG_ONLY( Node* disq_node = NULL; )
    bool  can_eliminate = true;
  
+   Unique_Node_List worklist;
    Node* res = alloc->result_cast();
    const TypeOopPtr* res_type = NULL;
    if (res == NULL) {
      // All users were eliminated.
    } else if (!res->is_CheckCastPP()) {
      NOT_PRODUCT(fail_eliminate = "Allocation does not have unique CheckCastPP";)
      can_eliminate = false;
    } else {
+     worklist.push(res);
      res_type = _igvn.type(res)->isa_oopptr();
      if (res_type == NULL) {
        NOT_PRODUCT(fail_eliminate = "Neither instance or array allocation";)
        can_eliminate = false;
      } else if (res_type->isa_aryptr()) {

*** 573,21 ***
          can_eliminate = false;
        }
      }
    }
  
!   if (can_eliminate && res != NULL) {
!     for (DUIterator_Fast jmax, j = res->fast_outs(jmax);
!                                j < jmax && can_eliminate; j++) {
        Node* use = res->fast_out(j);
  
        if (use->is_AddP()) {
          const TypePtr* addp_type = _igvn.type(use)->is_ptr();
          int offset = addp_type->offset();
  
          if (offset == Type::OffsetTop || offset == Type::OffsetBot) {
!           NOT_PRODUCT(fail_eliminate = "Undefined field referrence";)
            can_eliminate = false;
            break;
          }
          for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
                                     k < kmax && can_eliminate; k++) {
--- 634,21 ---
          can_eliminate = false;
        }
      }
    }
  
!   while (can_eliminate && worklist.size() > 0) {
!     res = worklist.pop();
!     for (DUIterator_Fast jmax, j = res->fast_outs(jmax); j < jmax && can_eliminate; j++) {
        Node* use = res->fast_out(j);
  
        if (use->is_AddP()) {
          const TypePtr* addp_type = _igvn.type(use)->is_ptr();
          int offset = addp_type->offset();
  
          if (offset == Type::OffsetTop || offset == Type::OffsetBot) {
!           NOT_PRODUCT(fail_eliminate = "Undefined field reference";)
            can_eliminate = false;
            break;
          }
          for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
                                     k < kmax && can_eliminate; k++) {

*** 596,11 ***
                SHENANDOAHGC_ONLY(&& (!UseShenandoahGC || !ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(n))) ) {
              DEBUG_ONLY(disq_node = n;)
              if (n->is_Load() || n->is_LoadStore()) {
                NOT_PRODUCT(fail_eliminate = "Field load";)
              } else {
!               NOT_PRODUCT(fail_eliminate = "Not store field referrence";)
              }
              can_eliminate = false;
            }
          }
        } else if (use->is_ArrayCopy() &&
--- 657,11 ---
                SHENANDOAHGC_ONLY(&& (!UseShenandoahGC || !ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(n))) ) {
              DEBUG_ONLY(disq_node = n;)
              if (n->is_Load() || n->is_LoadStore()) {
                NOT_PRODUCT(fail_eliminate = "Field load";)
              } else {
!               NOT_PRODUCT(fail_eliminate = "Not store field reference";)
              }
              can_eliminate = false;
            }
          }
        } else if (use->is_ArrayCopy() &&

*** 624,10 ***
--- 685,18 ---
            NOT_PRODUCT(fail_eliminate = "NULL or TOP memory";)
            can_eliminate = false;
          } else {
            safepoints.append_if_missing(sfpt);
          }
+       } else if (use->is_InlineType() && use->isa_InlineType()->get_oop() == res) {
+         // ok to eliminate
+       } else if (use->is_InlineTypePtr() && use->isa_InlineTypePtr()->get_oop() == res) {
+         // Process users
+         worklist.push(use);
+       } else if (use->Opcode() == Op_StoreX && use->in(MemNode::Address) == res) {
+         // Store to mark word of inline type larval buffer
+         assert(res_type->is_inlinetypeptr(), "Unexpected store to mark word");
        } else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark
          if (use->is_Phi()) {
            if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) {
              NOT_PRODUCT(fail_eliminate = "Object is return value";)
            } else {

*** 635,16 ***
            }
            DEBUG_ONLY(disq_node = use;)
          } else {
            if (use->Opcode() == Op_Return) {
              NOT_PRODUCT(fail_eliminate = "Object is return value";)
!           }else {
              NOT_PRODUCT(fail_eliminate = "Object is referenced by node";)
            }
            DEBUG_ONLY(disq_node = use;)
          }
          can_eliminate = false;
        }
      }
    }
  
  #ifndef PRODUCT
--- 704,19 ---
            }
            DEBUG_ONLY(disq_node = use;)
          } else {
            if (use->Opcode() == Op_Return) {
              NOT_PRODUCT(fail_eliminate = "Object is return value";)
!           } else {
              NOT_PRODUCT(fail_eliminate = "Object is referenced by node";)
            }
            DEBUG_ONLY(disq_node = use;)
          }
          can_eliminate = false;
+       } else {
+         assert(use->Opcode() == Op_CastP2X, "should be");
+         assert(!use->has_out_with(Op_OrL), "should have been removed because oop is never null");
        }
      }
    }
  
  #ifndef PRODUCT

*** 653,11 ***
        tty->print("Scalar ");
        if (res == NULL)
          alloc->dump();
        else
          res->dump();
!     } else if (alloc->_is_scalar_replaceable) {
        tty->print("NotScalar (%s)", fail_eliminate);
        if (res == NULL)
          alloc->dump();
        else
          res->dump();
--- 725,11 ---
        tty->print("Scalar ");
        if (res == NULL)
          alloc->dump();
        else
          res->dump();
!     } else {
        tty->print("NotScalar (%s)", fail_eliminate);
        if (res == NULL)
          alloc->dump();
        else
          res->dump();

*** 703,17 ***
--- 775,27 ---
        // find the array's elements which will be needed for safepoint debug information
        nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1);
        assert(klass->is_array_klass() && nfields >= 0, "must be an array klass.");
        elem_type = klass->as_array_klass()->element_type();
        basic_elem_type = elem_type->basic_type();
+       if (elem_type->is_inlinetype() && !klass->is_flat_array_klass()) {
+         assert(basic_elem_type == T_INLINE_TYPE, "unexpected element basic type");
+         basic_elem_type = T_OBJECT;
+       }
        array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
        element_size = type2aelembytes(basic_elem_type);
+       if (klass->is_flat_array_klass()) {
+         // Flattened inline type array
+         element_size = klass->as_flat_array_klass()->element_byte_size();
+       }
      }
    }
    //
    // Process the safepoint uses
    //
+   assert(safepoints.length() == 0 || !res_type->is_inlinetypeptr(), "Inline type allocations should not have safepoint uses");
+   Unique_Node_List value_worklist;
    while (safepoints.length() > 0) {
      SafePointNode* sfpt = safepoints.pop();
      Node* mem = sfpt->memory();
      Node* ctl = sfpt->control();
      assert(sfpt->jvms() != NULL, "missed JVMS");

*** 736,10 ***
--- 818,11 ---
        if (iklass != NULL) {
          field = iklass->nonstatic_field_at(j);
          offset = field->offset();
          elem_type = field->type();
          basic_elem_type = field->layout_type();
+         assert(!field->is_flattened(), "flattened inline type fields should not have safepoint uses");
        } else {
          offset = array_base + j * (intptr_t)element_size;
        }
  
        const Type *field_type;

*** 763,13 ***
          }
        } else {
          field_type = Type::get_const_basic_type(basic_elem_type);
        }
  
!       const TypeOopPtr *field_addr_type = res_type->add_offset(offset)->isa_oopptr();
! 
!       Node *field_val = value_from_mem(mem, ctl, basic_elem_type, field_type, field_addr_type, alloc);
        if (field_val == NULL) {
          // We weren't able to find a value for this field,
          // give up on eliminating this allocation.
  
          // Remove any extra entries we added to the safepoint.
--- 846,19 ---
          }
        } else {
          field_type = Type::get_const_basic_type(basic_elem_type);
        }
  
!       Node* field_val = NULL;
!       const TypeOopPtr* field_addr_type = res_type->add_offset(offset)->isa_oopptr();
!       if (klass->is_flat_array_klass()) {
+         ciInlineKlass* vk = elem_type->as_inline_klass();
+         assert(vk->flatten_array(), "must be flattened");
+         field_val = inline_type_from_mem(mem, ctl, vk, field_addr_type->isa_aryptr(), 0, alloc);
+       } else {
+         field_val = value_from_mem(mem, ctl, basic_elem_type, field_type, field_addr_type, alloc);
+       }
        if (field_val == NULL) {
          // We weren't able to find a value for this field,
          // give up on eliminating this allocation.
  
          // Remove any extra entries we added to the safepoint.

*** 828,14 ***
        if (UseCompressedOops && field_type->isa_narrowoop()) {
          // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
          // to be able scalar replace the allocation.
          if (field_val->is_EncodeP()) {
            field_val = field_val->in(1);
!         } else {
            field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
          }
        }
        sfpt->add_req(field_val);
      }
      JVMState *jvms = sfpt->jvms();
      jvms->set_endoff(sfpt->req());
      // Now make a pass over the debug information replacing any references
--- 917,18 ---
        if (UseCompressedOops && field_type->isa_narrowoop()) {
          // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
          // to be able scalar replace the allocation.
          if (field_val->is_EncodeP()) {
            field_val = field_val->in(1);
!         } else if (!field_val->is_InlineTypeBase()) {
            field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
          }
        }
+       if (field_val->is_InlineTypeBase()) {
+         // Keep track of inline types to scalarize them later
+         value_worklist.push(field_val);
+       }
        sfpt->add_req(field_val);
      }
      JVMState *jvms = sfpt->jvms();
      jvms->set_endoff(sfpt->req());
      // Now make a pass over the debug information replacing any references

*** 844,10 ***
--- 937,18 ---
      int end   = jvms->debug_end();
      sfpt->replace_edges_in_range(res, sobj, start, end, &_igvn);
      _igvn._worklist.push(sfpt);
      safepoints_done.append_if_missing(sfpt); // keep it for rollback
    }
+   // Scalarize inline types that were added to the safepoint.
+   // Don't allow linking a constant oop (if available) for flat array elements
+   // because Deoptimization::reassign_flat_array_elements needs field values.
+   bool allow_oop = (klass == NULL) || !klass->is_flat_array_klass();
+   for (uint i = 0; i < value_worklist.size(); ++i) {
+     InlineTypeBaseNode* vt = value_worklist.at(i)->as_InlineTypeBase();
+     vt->make_scalar_in_safepoints(&_igvn, allow_oop);
+   }
    return true;
  }
  
  static void disconnect_projections(MultiNode* n, PhaseIterGVN& igvn) {
    Node* ctl_proj = n->proj_out_or_null(TypeFunc::Control);

*** 859,34 ***
      igvn.replace_node(mem_proj, n->in(TypeFunc::Memory));
    }
  }
  
  // Process users of eliminated allocation.
! void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
    Node* res = alloc->result_cast();
    if (res != NULL) {
      for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) {
        Node *use = res->last_out(j);
        uint oc1 = res->outcnt();
  
        if (use->is_AddP()) {
          for (DUIterator_Last kmin, k = use->last_outs(kmin); k >= kmin; ) {
            Node *n = use->last_out(k);
            uint oc2 = use->outcnt();
            if (n->is_Store()) {
! #ifdef ASSERT
!             // Verify that there is no dependent MemBarVolatile nodes,
!             // they should be removed during IGVN, see MemBarNode::Ideal().
!             for (DUIterator_Fast pmax, p = n->fast_outs(pmax);
!                                        p < pmax; p++) {
!               Node* mb = n->fast_out(p);
!               assert(mb->is_Initialize() || !mb->is_MemBar() ||
-                      mb->req() <= MemBarNode::Precedent ||
-                      mb->in(MemBarNode::Precedent) != n,
-                      "MemBarVolatile should be eliminated for non-escaping object");
              }
- #endif
              _igvn.replace_node(n, n->in(MemNode::Memory));
            } else {
              eliminate_gc_barrier(n);
            }
            k -= (oc2 - use->outcnt());
--- 960,35 ---
      igvn.replace_node(mem_proj, n->in(TypeFunc::Memory));
    }
  }
  
  // Process users of eliminated allocation.
! void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc, bool inline_alloc) {
+   Unique_Node_List worklist;
    Node* res = alloc->result_cast();
    if (res != NULL) {
+     worklist.push(res);
+   }
+   while (worklist.size() > 0) {
+     res = worklist.pop();
      for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) {
        Node *use = res->last_out(j);
        uint oc1 = res->outcnt();
  
        if (use->is_AddP()) {
          for (DUIterator_Last kmin, k = use->last_outs(kmin); k >= kmin; ) {
            Node *n = use->last_out(k);
            uint oc2 = use->outcnt();
            if (n->is_Store()) {
!             for (DUIterator_Fast pmax, p = n->fast_outs(pmax); p < pmax; p++) {
!               MemBarNode* mb = n->fast_out(p)->isa_MemBar();
!               if (mb != NULL && mb->req() <= MemBarNode::Precedent && mb->in(MemBarNode::Precedent) == n) {
!                 // MemBarVolatiles should have been removed by MemBarNode::Ideal() for non-inline allocations
!                 assert(inline_alloc, "MemBarVolatile should be eliminated for non-escaping object");
!                 mb->remove(&_igvn);
!               }
              }
              _igvn.replace_node(n, n->in(MemNode::Memory));
            } else {
              eliminate_gc_barrier(n);
            }
            k -= (oc2 - use->outcnt());

*** 906,16 ***
            }
          } else {
            assert(ac->is_arraycopy_validated() ||
                   ac->is_copyof_validated() ||
                   ac->is_copyofrange_validated(), "unsupported");
!           CallProjections callprojs;
-           ac->extract_projections(&callprojs, true);
  
!           _igvn.replace_node(callprojs.fallthrough_ioproj, ac->in(TypeFunc::I_O));
!           _igvn.replace_node(callprojs.fallthrough_memproj, ac->in(TypeFunc::Memory));
!           _igvn.replace_node(callprojs.fallthrough_catchproj, ac->in(TypeFunc::Control));
  
            // Set control to top. IGVN will remove the remaining projections
            ac->set_req(0, top());
            ac->replace_edge(res, top(), &_igvn);
  
--- 1008,15 ---
            }
          } else {
            assert(ac->is_arraycopy_validated() ||
                   ac->is_copyof_validated() ||
                   ac->is_copyofrange_validated(), "unsupported");
!           CallProjections* callprojs = ac->extract_projections(true);
  
!           _igvn.replace_node(callprojs->fallthrough_ioproj, ac->in(TypeFunc::I_O));
!           _igvn.replace_node(callprojs->fallthrough_memproj, ac->in(TypeFunc::Memory));
!           _igvn.replace_node(callprojs->fallthrough_catchproj, ac->in(TypeFunc::Control));
  
            // Set control to top. IGVN will remove the remaining projections
            ac->set_req(0, top());
            ac->replace_edge(res, top(), &_igvn);
  

*** 928,10 ***
--- 1029,24 ---
            if (src->outcnt() == 0 && !src->is_top()) {
              _igvn.remove_dead_node(src);
            }
          }
          _igvn._worklist.push(ac);
+       } else if (use->is_InlineType()) {
+         assert(use->isa_InlineType()->get_oop() == res, "unexpected inline type use");
+         _igvn.rehash_node_delayed(use);
+         use->isa_InlineType()->set_oop(_igvn.zerocon(T_INLINE_TYPE));
+       } else if (use->is_InlineTypePtr()) {
+         assert(use->isa_InlineTypePtr()->get_oop() == res, "unexpected inline type ptr use");
+         _igvn.rehash_node_delayed(use);
+         use->isa_InlineTypePtr()->set_oop(_igvn.zerocon(T_INLINE_TYPE));
+         // Process users
+         worklist.push(use);
+       } else if (use->Opcode() == Op_StoreX && use->in(MemNode::Address) == res) {
+         // Store to mark word of inline type larval buffer
+         assert(inline_alloc, "Unexpected store to mark word");
+         _igvn.replace_node(use, use->in(MemNode::Memory));
        } else {
          eliminate_gc_barrier(use);
        }
        j -= (oc1 - res->outcnt());
      }

*** 940,110 ***
    }
  
    //
    // Process other users of allocation's projections
    //
!   if (_callprojs.resproj != NULL && _callprojs.resproj->outcnt() != 0) {
      // First disconnect stores captured by Initialize node.
      // If Initialize node is eliminated first in the following code,
      // it will kill such stores and DUIterator_Last will assert.
!     for (DUIterator_Fast jmax, j = _callprojs.resproj->fast_outs(jmax);  j < jmax; j++) {
!       Node* use = _callprojs.resproj->fast_out(j);
        if (use->is_AddP()) {
          // raw memory addresses used only by the initialization
          _igvn.replace_node(use, C->top());
          --j; --jmax;
        }
      }
!     for (DUIterator_Last jmin, j = _callprojs.resproj->last_outs(jmin); j >= jmin; ) {
!       Node* use = _callprojs.resproj->last_out(j);
!       uint oc1 = _callprojs.resproj->outcnt();
        if (use->is_Initialize()) {
          // Eliminate Initialize node.
          InitializeNode *init = use->as_Initialize();
          assert(init->outcnt() <= 2, "only a control and memory projection expected");
          Node *ctrl_proj = init->proj_out_or_null(TypeFunc::Control);
          if (ctrl_proj != NULL) {
            _igvn.replace_node(ctrl_proj, init->in(TypeFunc::Control));
  #ifdef ASSERT
            // If the InitializeNode has no memory out, it will die, and tmp will become NULL
            Node* tmp = init->in(TypeFunc::Control);
!           assert(tmp == NULL || tmp == _callprojs.fallthrough_catchproj, "allocation control projection");
  #endif
          }
          Node *mem_proj = init->proj_out_or_null(TypeFunc::Memory);
          if (mem_proj != NULL) {
            Node *mem = init->in(TypeFunc::Memory);
  #ifdef ASSERT
            if (mem->is_MergeMem()) {
!             assert(mem->in(TypeFunc::Memory) == _callprojs.fallthrough_memproj, "allocation memory projection");
            } else {
!             assert(mem == _callprojs.fallthrough_memproj, "allocation memory projection");
            }
  #endif
            _igvn.replace_node(mem_proj, mem);
          }
        } else  {
          assert(false, "only Initialize or AddP expected");
        }
!       j -= (oc1 - _callprojs.resproj->outcnt());
      }
    }
!   if (_callprojs.fallthrough_catchproj != NULL) {
!     _igvn.replace_node(_callprojs.fallthrough_catchproj, alloc->in(TypeFunc::Control));
    }
!   if (_callprojs.fallthrough_memproj != NULL) {
!     _igvn.replace_node(_callprojs.fallthrough_memproj, alloc->in(TypeFunc::Memory));
    }
!   if (_callprojs.catchall_memproj != NULL) {
!     _igvn.replace_node(_callprojs.catchall_memproj, C->top());
    }
!   if (_callprojs.fallthrough_ioproj != NULL) {
!     _igvn.replace_node(_callprojs.fallthrough_ioproj, alloc->in(TypeFunc::I_O));
    }
!   if (_callprojs.catchall_ioproj != NULL) {
!     _igvn.replace_node(_callprojs.catchall_ioproj, C->top());
    }
!   if (_callprojs.catchall_catchproj != NULL) {
!     _igvn.replace_node(_callprojs.catchall_catchproj, C->top());
    }
  }
  
  bool PhaseMacroExpand::eliminate_allocate_node(AllocateNode *alloc) {
    // If reallocation fails during deoptimization we'll pop all
    // interpreter frames for this compiled frame and that won't play
    // nice with JVMTI popframe.
    // We avoid this issue by eager reallocation when the popframe request
    // is received.
!   if (!EliminateAllocations || !alloc->_is_non_escaping) {
      return false;
    }
    Node* klass = alloc->in(AllocateNode::KlassNode);
    const TypeKlassPtr* tklass = _igvn.type(klass)->is_klassptr();
!   Node* res = alloc->result_cast();
    // Eliminate boxing allocations which are not used
!   // regardless scalar replacable status.
!   bool boxing_alloc = C->eliminate_boxing() &&
!                       tklass->klass()->is_instance_klass()  &&
                        tklass->klass()->as_instance_klass()->is_box_klass();
!   if (!alloc->_is_scalar_replaceable && (!boxing_alloc || (res != NULL))) {
      return false;
    }
  
!   alloc->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/);
  
    GrowableArray <SafePointNode *> safepoints;
    if (!can_eliminate_allocation(alloc, safepoints)) {
      return false;
    }
  
    if (!alloc->_is_scalar_replaceable) {
!     assert(res == NULL, "sanity");
      // We can only eliminate allocation if all debug info references
      // are already replaced with SafePointScalarObject because
      // we can't search for a fields value without instance_id.
      if (safepoints.length() > 0) {
        return false;
      }
    }
  
    if (!scalar_replacement(alloc, safepoints)) {
--- 1055,122 ---
    }
  
    //
    // Process other users of allocation's projections
    //
!   if (_callprojs->resproj[0] != NULL && _callprojs->resproj[0]->outcnt() != 0) {
      // First disconnect stores captured by Initialize node.
      // If Initialize node is eliminated first in the following code,
      // it will kill such stores and DUIterator_Last will assert.
!     for (DUIterator_Fast jmax, j = _callprojs->resproj[0]->fast_outs(jmax);  j < jmax; j++) {
!       Node* use = _callprojs->resproj[0]->fast_out(j);
        if (use->is_AddP()) {
          // raw memory addresses used only by the initialization
          _igvn.replace_node(use, C->top());
          --j; --jmax;
        }
      }
!     for (DUIterator_Last jmin, j = _callprojs->resproj[0]->last_outs(jmin); j >= jmin; ) {
!       Node* use = _callprojs->resproj[0]->last_out(j);
!       uint oc1 = _callprojs->resproj[0]->outcnt();
        if (use->is_Initialize()) {
          // Eliminate Initialize node.
          InitializeNode *init = use->as_Initialize();
          assert(init->outcnt() <= 2, "only a control and memory projection expected");
          Node *ctrl_proj = init->proj_out_or_null(TypeFunc::Control);
          if (ctrl_proj != NULL) {
            _igvn.replace_node(ctrl_proj, init->in(TypeFunc::Control));
  #ifdef ASSERT
            // If the InitializeNode has no memory out, it will die, and tmp will become NULL
            Node* tmp = init->in(TypeFunc::Control);
!           assert(tmp == NULL || tmp == _callprojs->fallthrough_catchproj, "allocation control projection");
  #endif
          }
          Node *mem_proj = init->proj_out_or_null(TypeFunc::Memory);
          if (mem_proj != NULL) {
            Node *mem = init->in(TypeFunc::Memory);
  #ifdef ASSERT
            if (mem->is_MergeMem()) {
!             assert(mem->in(TypeFunc::Memory) == _callprojs->fallthrough_memproj, "allocation memory projection");
            } else {
!             assert(mem == _callprojs->fallthrough_memproj, "allocation memory projection");
            }
  #endif
            _igvn.replace_node(mem_proj, mem);
          }
+       } else if (use->Opcode() == Op_MemBarStoreStore) {
+         // Inline type buffer allocations are followed by a membar
+         assert(inline_alloc, "Unexpected MemBarStoreStore");
+         use->as_MemBar()->remove(&_igvn);
        } else  {
          assert(false, "only Initialize or AddP expected");
        }
!       j -= (oc1 - _callprojs->resproj[0]->outcnt());
      }
    }
!   if (_callprojs->fallthrough_catchproj != NULL) {
!     _igvn.replace_node(_callprojs->fallthrough_catchproj, alloc->in(TypeFunc::Control));
    }
!   if (_callprojs->fallthrough_memproj != NULL) {
!     _igvn.replace_node(_callprojs->fallthrough_memproj, alloc->in(TypeFunc::Memory));
    }
!   if (_callprojs->catchall_memproj != NULL) {
!     _igvn.replace_node(_callprojs->catchall_memproj, C->top());
    }
!   if (_callprojs->fallthrough_ioproj != NULL) {
!     _igvn.replace_node(_callprojs->fallthrough_ioproj, alloc->in(TypeFunc::I_O));
    }
!   if (_callprojs->catchall_ioproj != NULL) {
!     _igvn.replace_node(_callprojs->catchall_ioproj, C->top());
    }
!   if (_callprojs->catchall_catchproj != NULL) {
!     _igvn.replace_node(_callprojs->catchall_catchproj, C->top());
    }
  }
  
  bool PhaseMacroExpand::eliminate_allocate_node(AllocateNode *alloc) {
    // If reallocation fails during deoptimization we'll pop all
    // interpreter frames for this compiled frame and that won't play
    // nice with JVMTI popframe.
    // We avoid this issue by eager reallocation when the popframe request
    // is received.
!   if (!EliminateAllocations) {
      return false;
    }
    Node* klass = alloc->in(AllocateNode::KlassNode);
    const TypeKlassPtr* tklass = _igvn.type(klass)->is_klassptr();
! 
+   // Attempt to eliminate inline type buffer allocations
+   // regardless of usage and escape/replaceable status.
+   bool inline_alloc = tklass->klass()->is_inlinetype();
+   if (!alloc->_is_non_escaping && !inline_alloc) {
+     return false;
+   }
    // Eliminate boxing allocations which are not used
!   // regardless of scalar replaceable status.
!   Node* res = alloc->result_cast();
!   bool boxing_alloc = (res == NULL) && C->eliminate_boxing() &&
+                       tklass->klass()->is_instance_klass() &&
                        tklass->klass()->as_instance_klass()->is_box_klass();
!   if (!alloc->_is_scalar_replaceable && !boxing_alloc && !inline_alloc) {
      return false;
    }
  
!   _callprojs = alloc->extract_projections(false /*separate_io_proj*/, false /*do_asserts*/);
  
    GrowableArray <SafePointNode *> safepoints;
    if (!can_eliminate_allocation(alloc, safepoints)) {
      return false;
    }
  
    if (!alloc->_is_scalar_replaceable) {
!     assert(res == NULL || inline_alloc, "sanity");
      // We can only eliminate allocation if all debug info references
      // are already replaced with SafePointScalarObject because
      // we can't search for a fields value without instance_id.
      if (safepoints.length() > 0) {
+       assert(!inline_alloc, "Inline type allocations should not have safepoint uses");
        return false;
      }
    }
  
    if (!scalar_replacement(alloc, safepoints)) {

*** 1060,11 ***
        p = p->caller();
      }
      log->tail("eliminate_allocation");
    }
  
!   process_users_of_allocation(alloc);
  
  #ifndef PRODUCT
    if (PrintEliminateAllocations) {
      if (alloc->is_AllocateArray())
        tty->print_cr("++++ Eliminated: %d AllocateArray", alloc->_idx);
--- 1187,11 ---
        p = p->caller();
      }
      log->tail("eliminate_allocation");
    }
  
!   process_users_of_allocation(alloc, inline_alloc);
  
  #ifndef PRODUCT
    if (PrintEliminateAllocations) {
      if (alloc->is_AllocateArray())
        tty->print_cr("++++ Eliminated: %d AllocateArray", alloc->_idx);

*** 1082,13 ***
      return false;
    }
  
    assert(boxing->result_cast() == NULL, "unexpected boxing node result");
  
!   boxing->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/);
  
!   const TypeTuple* r = boxing->tf()->range();
    assert(r->cnt() > TypeFunc::Parms, "sanity");
    const TypeInstPtr* t = r->field_at(TypeFunc::Parms)->isa_instptr();
    assert(t != NULL, "sanity");
  
    CompileLog* log = C->log();
--- 1209,13 ---
      return false;
    }
  
    assert(boxing->result_cast() == NULL, "unexpected boxing node result");
  
!   _callprojs = boxing->extract_projections(false /*separate_io_proj*/, false /*do_asserts*/);
  
!   const TypeTuple* r = boxing->tf()->range_sig();
    assert(r->cnt() > TypeFunc::Parms, "sanity");
    const TypeInstPtr* t = r->field_at(TypeFunc::Parms)->isa_instptr();
    assert(t != NULL, "sanity");
  
    CompileLog* log = C->log();

*** 1285,17 ***
      slow_region = new RegionNode(3);
  
      // Now make the initial failure test.  Usually a too-big test but
      // might be a TRUE for finalizers or a fancy class check for
      // newInstance0.
!     IfNode *toobig_iff = new IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
      transform_later(toobig_iff);
      // Plug the failing-too-big test into the slow-path region
!     Node *toobig_true = new IfTrueNode( toobig_iff );
      transform_later(toobig_true);
      slow_region    ->init_req( too_big_or_final_path, toobig_true );
!     toobig_false = new IfFalseNode( toobig_iff );
      transform_later(toobig_false);
    } else {
      // No initial test, just fall into next case
      assert(allocation_has_use || !expand_fast_path, "Should already have been handled");
      toobig_false = ctrl;
--- 1412,17 ---
      slow_region = new RegionNode(3);
  
      // Now make the initial failure test.  Usually a too-big test but
      // might be a TRUE for finalizers or a fancy class check for
      // newInstance0.
!     IfNode* toobig_iff = new IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
      transform_later(toobig_iff);
      // Plug the failing-too-big test into the slow-path region
!     Node* toobig_true = new IfTrueNode(toobig_iff);
      transform_later(toobig_true);
      slow_region    ->init_req( too_big_or_final_path, toobig_true );
!     toobig_false = new IfFalseNode(toobig_iff);
      transform_later(toobig_false);
    } else {
      // No initial test, just fall into next case
      assert(allocation_has_use || !expand_fast_path, "Should already have been handled");
      toobig_false = ctrl;

*** 1330,10 ***
--- 1457,11 ---
      result_phi_i_o->init_req(slow_result_path, i_o);
  
      // Name successful fast-path variables
      Node* fast_oop_ctrl;
      Node* fast_oop_rawmem;
+ 
      if (allocation_has_use) {
        Node* needgc_ctrl = NULL;
        result_phi_rawoop = new PhiNode(result_region, TypeRawPtr::BOTTOM);
  
        intx prefetch_lines = length != NULL ? AllocatePrefetchLines : AllocateInstancePrefetchLines;

*** 1387,10 ***
--- 1515,13 ---
    call->init_req(TypeFunc::FramePtr,  alloc->in(TypeFunc::FramePtr));
  
    call->init_req(TypeFunc::Parms+0, klass_node);
    if (length != NULL) {
      call->init_req(TypeFunc::Parms+1, length);
+   } else {
+     // Let the runtime know if this is a larval allocation
+     call->init_req(TypeFunc::Parms+1, _igvn.intcon(alloc->_larval));
    }
  
    // Copy debug information and adjust JVMState information, then replace
    // allocate node with the call
    call->copy_call_debug_info(&_igvn, alloc);

*** 1412,47 ***
    //  Allocate                   Catch
    //        ^---Proj(io) <-------+   ^---CatchProj(io)
    //
    //  We are interested in the CatchProj nodes.
    //
!   call->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/);
  
    // An allocate node has separate memory projections for the uses on
    // the control and i_o paths. Replace the control memory projection with
    // result_phi_rawmem (unless we are only generating a slow call when
    // both memory projections are combined)
!   if (expand_fast_path && _callprojs.fallthrough_memproj != NULL) {
!     migrate_outs(_callprojs.fallthrough_memproj, result_phi_rawmem);
    }
    // Now change uses of catchall_memproj to use fallthrough_memproj and delete
    // catchall_memproj so we end up with a call that has only 1 memory projection.
!   if (_callprojs.catchall_memproj != NULL ) {
!     if (_callprojs.fallthrough_memproj == NULL) {
!       _callprojs.fallthrough_memproj = new ProjNode(call, TypeFunc::Memory);
!       transform_later(_callprojs.fallthrough_memproj);
      }
!     migrate_outs(_callprojs.catchall_memproj, _callprojs.fallthrough_memproj);
!     _igvn.remove_dead_node(_callprojs.catchall_memproj);
    }
  
    // An allocate node has separate i_o projections for the uses on the control
    // and i_o paths. Always replace the control i_o projection with result i_o
    // otherwise incoming i_o become dead when only a slow call is generated
    // (it is different from memory projections where both projections are
    // combined in such case).
!   if (_callprojs.fallthrough_ioproj != NULL) {
!     migrate_outs(_callprojs.fallthrough_ioproj, result_phi_i_o);
    }
    // Now change uses of catchall_ioproj to use fallthrough_ioproj and delete
    // catchall_ioproj so we end up with a call that has only 1 i_o projection.
!   if (_callprojs.catchall_ioproj != NULL ) {
!     if (_callprojs.fallthrough_ioproj == NULL) {
!       _callprojs.fallthrough_ioproj = new ProjNode(call, TypeFunc::I_O);
!       transform_later(_callprojs.fallthrough_ioproj);
      }
!     migrate_outs(_callprojs.catchall_ioproj, _callprojs.fallthrough_ioproj);
!     _igvn.remove_dead_node(_callprojs.catchall_ioproj);
    }
  
    // if we generated only a slow call, we are done
    if (!expand_fast_path) {
      // Now we can unhook i_o.
--- 1543,47 ---
    //  Allocate                   Catch
    //        ^---Proj(io) <-------+   ^---CatchProj(io)
    //
    //  We are interested in the CatchProj nodes.
    //
!   _callprojs = call->extract_projections(false /*separate_io_proj*/, false /*do_asserts*/);
  
    // An allocate node has separate memory projections for the uses on
    // the control and i_o paths. Replace the control memory projection with
    // result_phi_rawmem (unless we are only generating a slow call when
    // both memory projections are combined)
!   if (expand_fast_path && _callprojs->fallthrough_memproj != NULL) {
!     _igvn.replace_in_uses(_callprojs->fallthrough_memproj, result_phi_rawmem);
    }
    // Now change uses of catchall_memproj to use fallthrough_memproj and delete
    // catchall_memproj so we end up with a call that has only 1 memory projection.
!   if (_callprojs->catchall_memproj != NULL) {
!     if (_callprojs->fallthrough_memproj == NULL) {
!       _callprojs->fallthrough_memproj = new ProjNode(call, TypeFunc::Memory);
!       transform_later(_callprojs->fallthrough_memproj);
      }
!     _igvn.replace_in_uses(_callprojs->catchall_memproj, _callprojs->fallthrough_memproj);
!     _igvn.remove_dead_node(_callprojs->catchall_memproj);
    }
  
    // An allocate node has separate i_o projections for the uses on the control
    // and i_o paths. Always replace the control i_o projection with result i_o
    // otherwise incoming i_o become dead when only a slow call is generated
    // (it is different from memory projections where both projections are
    // combined in such case).
!   if (_callprojs->fallthrough_ioproj != NULL) {
!     _igvn.replace_in_uses(_callprojs->fallthrough_ioproj, result_phi_i_o);
    }
    // Now change uses of catchall_ioproj to use fallthrough_ioproj and delete
    // catchall_ioproj so we end up with a call that has only 1 i_o projection.
!   if (_callprojs->catchall_ioproj != NULL) {
!     if (_callprojs->fallthrough_ioproj == NULL) {
!       _callprojs->fallthrough_ioproj = new ProjNode(call, TypeFunc::I_O);
!       transform_later(_callprojs->fallthrough_ioproj);
      }
!     _igvn.replace_in_uses(_callprojs->catchall_ioproj, _callprojs->fallthrough_ioproj);
!     _igvn.remove_dead_node(_callprojs->catchall_ioproj);
    }
  
    // if we generated only a slow call, we are done
    if (!expand_fast_path) {
      // Now we can unhook i_o.

*** 1467,35 ***
        // Leave i_o attached to this call to avoid problems in preceding graph.
      }
      return;
    }
  
!   if (_callprojs.fallthrough_catchproj != NULL) {
!     ctrl = _callprojs.fallthrough_catchproj->clone();
      transform_later(ctrl);
!     _igvn.replace_node(_callprojs.fallthrough_catchproj, result_region);
    } else {
      ctrl = top();
    }
    Node *slow_result;
!   if (_callprojs.resproj == NULL) {
      // no uses of the allocation result
      slow_result = top();
    } else {
!     slow_result = _callprojs.resproj->clone();
      transform_later(slow_result);
!     _igvn.replace_node(_callprojs.resproj, result_phi_rawoop);
    }
  
    // Plug slow-path into result merge point
    result_region->init_req( slow_result_path, ctrl);
    transform_later(result_region);
    if (allocation_has_use) {
      result_phi_rawoop->init_req(slow_result_path, slow_result);
      transform_later(result_phi_rawoop);
    }
!   result_phi_rawmem->init_req(slow_result_path, _callprojs.fallthrough_memproj);
    transform_later(result_phi_rawmem);
    transform_later(result_phi_i_o);
    // This completes all paths into the result merge point
  }
  
--- 1598,35 ---
        // Leave i_o attached to this call to avoid problems in preceding graph.
      }
      return;
    }
  
!   if (_callprojs->fallthrough_catchproj != NULL) {
!     ctrl = _callprojs->fallthrough_catchproj->clone();
      transform_later(ctrl);
!     _igvn.replace_node(_callprojs->fallthrough_catchproj, result_region);
    } else {
      ctrl = top();
    }
    Node *slow_result;
!   if (_callprojs->resproj[0] == NULL) {
      // no uses of the allocation result
      slow_result = top();
    } else {
!     slow_result = _callprojs->resproj[0]->clone();
      transform_later(slow_result);
!     _igvn.replace_node(_callprojs->resproj[0], result_phi_rawoop);
    }
  
    // Plug slow-path into result merge point
    result_region->init_req( slow_result_path, ctrl);
    transform_later(result_region);
    if (allocation_has_use) {
      result_phi_rawoop->init_req(slow_result_path, slow_result);
      transform_later(result_phi_rawoop);
    }
!   result_phi_rawmem->init_req(slow_result_path, _callprojs->fallthrough_memproj);
    transform_later(result_phi_rawmem);
    transform_later(result_phi_i_o);
    // This completes all paths into the result merge point
  }
  

*** 1503,49 ***
  void PhaseMacroExpand::yank_alloc_node(AllocateNode* alloc) {
    Node* ctrl = alloc->in(TypeFunc::Control);
    Node* mem  = alloc->in(TypeFunc::Memory);
    Node* i_o  = alloc->in(TypeFunc::I_O);
  
!   alloc->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/);
!   if (_callprojs.resproj != NULL) {
!     for (DUIterator_Fast imax, i = _callprojs.resproj->fast_outs(imax); i < imax; i++) {
!       Node* use = _callprojs.resproj->fast_out(i);
        use->isa_MemBar()->remove(&_igvn);
        --imax;
        --i; // back up iterator
      }
!     assert(_callprojs.resproj->outcnt() == 0, "all uses must be deleted");
!     _igvn.remove_dead_node(_callprojs.resproj);
    }
!   if (_callprojs.fallthrough_catchproj != NULL) {
!     migrate_outs(_callprojs.fallthrough_catchproj, ctrl);
!     _igvn.remove_dead_node(_callprojs.fallthrough_catchproj);
    }
!   if (_callprojs.catchall_catchproj != NULL) {
!     _igvn.rehash_node_delayed(_callprojs.catchall_catchproj);
!     _callprojs.catchall_catchproj->set_req(0, top());
    }
!   if (_callprojs.fallthrough_proj != NULL) {
!     Node* catchnode = _callprojs.fallthrough_proj->unique_ctrl_out();
      _igvn.remove_dead_node(catchnode);
!     _igvn.remove_dead_node(_callprojs.fallthrough_proj);
    }
!   if (_callprojs.fallthrough_memproj != NULL) {
!     migrate_outs(_callprojs.fallthrough_memproj, mem);
!     _igvn.remove_dead_node(_callprojs.fallthrough_memproj);
    }
!   if (_callprojs.fallthrough_ioproj != NULL) {
!     migrate_outs(_callprojs.fallthrough_ioproj, i_o);
!     _igvn.remove_dead_node(_callprojs.fallthrough_ioproj);
    }
!   if (_callprojs.catchall_memproj != NULL) {
!     _igvn.rehash_node_delayed(_callprojs.catchall_memproj);
!     _callprojs.catchall_memproj->set_req(0, top());
    }
!   if (_callprojs.catchall_ioproj != NULL) {
!     _igvn.rehash_node_delayed(_callprojs.catchall_ioproj);
!     _callprojs.catchall_ioproj->set_req(0, top());
    }
  #ifndef PRODUCT
    if (PrintEliminateAllocations) {
      if (alloc->is_AllocateArray()) {
        tty->print_cr("++++ Eliminated: %d AllocateArray", alloc->_idx);
--- 1634,49 ---
  void PhaseMacroExpand::yank_alloc_node(AllocateNode* alloc) {
    Node* ctrl = alloc->in(TypeFunc::Control);
    Node* mem  = alloc->in(TypeFunc::Memory);
    Node* i_o  = alloc->in(TypeFunc::I_O);
  
!   _callprojs = alloc->extract_projections(false /*separate_io_proj*/, false /*do_asserts*/);
!   if (_callprojs->resproj[0] != NULL) {
!     for (DUIterator_Fast imax, i = _callprojs->resproj[0]->fast_outs(imax); i < imax; i++) {
!       Node* use = _callprojs->resproj[0]->fast_out(i);
        use->isa_MemBar()->remove(&_igvn);
        --imax;
        --i; // back up iterator
      }
!     assert(_callprojs->resproj[0]->outcnt() == 0, "all uses must be deleted");
!     _igvn.remove_dead_node(_callprojs->resproj[0]);
    }
!   if (_callprojs->fallthrough_catchproj != NULL) {
!     _igvn.replace_in_uses(_callprojs->fallthrough_catchproj, ctrl);
!     _igvn.remove_dead_node(_callprojs->fallthrough_catchproj);
    }
!   if (_callprojs->catchall_catchproj != NULL) {
!     _igvn.rehash_node_delayed(_callprojs->catchall_catchproj);
!     _callprojs->catchall_catchproj->set_req(0, top());
    }
!   if (_callprojs->fallthrough_proj != NULL) {
!     Node* catchnode = _callprojs->fallthrough_proj->unique_ctrl_out();
      _igvn.remove_dead_node(catchnode);
!     _igvn.remove_dead_node(_callprojs->fallthrough_proj);
    }
!   if (_callprojs->fallthrough_memproj != NULL) {
!     _igvn.replace_in_uses(_callprojs->fallthrough_memproj, mem);
!     _igvn.remove_dead_node(_callprojs->fallthrough_memproj);
    }
!   if (_callprojs->fallthrough_ioproj != NULL) {
!     _igvn.replace_in_uses(_callprojs->fallthrough_ioproj, i_o);
!     _igvn.remove_dead_node(_callprojs->fallthrough_ioproj);
    }
!   if (_callprojs->catchall_memproj != NULL) {
!     _igvn.rehash_node_delayed(_callprojs->catchall_memproj);
!     _callprojs->catchall_memproj->set_req(0, top());
    }
!   if (_callprojs->catchall_ioproj != NULL) {
!     _igvn.rehash_node_delayed(_callprojs->catchall_ioproj);
!     _callprojs->catchall_ioproj->set_req(0, top());
    }
  #ifndef PRODUCT
    if (PrintEliminateAllocations) {
      if (alloc->is_AllocateArray()) {
        tty->print_cr("++++ Eliminated: %d AllocateArray", alloc->_idx);

*** 1658,18 ***
    }
  }
  
  // Helper for PhaseMacroExpand::expand_allocate_common.
  // Initializes the newly-allocated storage.
! Node*
! PhaseMacroExpand::initialize_object(AllocateNode* alloc,
!                                     Node* control, Node* rawmem, Node* object,
!                                     Node* klass_node, Node* length,
-                                     Node* size_in_bytes) {
    InitializeNode* init = alloc->initialization();
    // Store the klass & mark bits
!   Node* mark_node = alloc->make_ideal_mark(&_igvn, object, control, rawmem);
    if (!mark_node->is_Con()) {
      transform_later(mark_node);
    }
    rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, TypeX_X->basic_type());
  
--- 1789,17 ---
    }
  }
  
  // Helper for PhaseMacroExpand::expand_allocate_common.
  // Initializes the newly-allocated storage.
! Node* PhaseMacroExpand::initialize_object(AllocateNode* alloc,
!                                           Node* control, Node* rawmem, Node* object,
!                                           Node* klass_node, Node* length,
!                                           Node* size_in_bytes) {
    InitializeNode* init = alloc->initialization();
    // Store the klass & mark bits
!   Node* mark_node = alloc->make_ideal_mark(&_igvn, control, rawmem);
    if (!mark_node->is_Con()) {
      transform_later(mark_node);
    }
    rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, TypeX_X->basic_type());
  

*** 1697,10 ***
--- 1827,12 ---
      // there can be two Allocates to one Initialize.  The answer in all these
      // edge cases is safety first.  It is always safe to clear immediately
      // within an Allocate, and then (maybe or maybe not) clear some more later.
      if (!(UseTLAB && ZeroTLAB)) {
        rawmem = ClearArrayNode::clear_memory(control, rawmem, object,
+                                             alloc->in(AllocateNode::DefaultValue),
+                                             alloc->in(AllocateNode::RawDefaultValue),
                                              header_size, size_in_bytes,
                                              &_igvn);
      }
    } else {
      if (!init->is_complete()) {

*** 2065,10 ***
--- 2197,53 ---
        }
      }
    }
  }
  
+ void PhaseMacroExpand::inline_type_guard(Node** ctrl, LockNode* lock) {
+   Node* obj = lock->obj_node();
+   const TypePtr* obj_type = _igvn.type(obj)->make_ptr();
+   if (!obj_type->can_be_inline_type()) {
+     return;
+   }
+   Node* mark = make_load(*ctrl, lock->memory(), obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
+   Node* value_mask = _igvn.MakeConX(markWord::inline_type_pattern);
+   Node* is_value = _igvn.transform(new AndXNode(mark, value_mask));
+   Node* cmp = _igvn.transform(new CmpXNode(is_value, value_mask));
+   Node* bol = _igvn.transform(new BoolNode(cmp, BoolTest::eq));
+   Node* unc_ctrl = generate_slow_guard(ctrl, bol, NULL);
+ 
+   int trap_request = Deoptimization::make_trap_request(Deoptimization::Reason_class_check, Deoptimization::Action_none);
+   address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point();
+   const TypePtr* no_memory_effects = NULL;
+   CallNode* unc = new CallStaticJavaNode(OptoRuntime::uncommon_trap_Type(), call_addr, "uncommon_trap",
+                                          no_memory_effects);
+   unc->init_req(TypeFunc::Control, unc_ctrl);
+   unc->init_req(TypeFunc::I_O, lock->i_o());
+   unc->init_req(TypeFunc::Memory, lock->memory());
+   unc->init_req(TypeFunc::FramePtr,  lock->in(TypeFunc::FramePtr));
+   unc->init_req(TypeFunc::ReturnAdr, lock->in(TypeFunc::ReturnAdr));
+   unc->init_req(TypeFunc::Parms+0, _igvn.intcon(trap_request));
+   unc->set_cnt(PROB_UNLIKELY_MAG(4));
+   unc->copy_call_debug_info(&_igvn, lock);
+ 
+   assert(unc->peek_monitor_box() == lock->box_node(), "wrong monitor");
+   assert((obj_type->is_inlinetypeptr() && unc->peek_monitor_obj()->is_SafePointScalarObject()) ||
+          (obj->is_InlineTypePtr() && obj->in(1) == unc->peek_monitor_obj()) ||
+          (obj == unc->peek_monitor_obj()), "wrong monitor");
+ 
+   // pop monitor and push obj back on stack: we trap before the monitorenter
+   unc->pop_monitor();
+   unc->grow_stack(unc->jvms(), 1);
+   unc->set_stack(unc->jvms(), unc->jvms()->stk_size()-1, obj);
+   _igvn.register_new_node_with_optimizer(unc);
+ 
+   unc_ctrl = _igvn.transform(new ProjNode(unc, TypeFunc::Control));
+   Node* halt = _igvn.transform(new HaltNode(unc_ctrl, lock->in(TypeFunc::FramePtr), "monitor enter on inline type"));
+   C->root()->add_req(halt);
+ }
+ 
  // we have determined that this lock/unlock can be eliminated, we simply
  // eliminate the node without expanding it.
  //
  // Note:  The membar's associated with the lock/unlock are currently not
  //        eliminated.  This should be investigated as a future enhancement.

*** 2096,25 ***
  
    Node* mem  = alock->in(TypeFunc::Memory);
    Node* ctrl = alock->in(TypeFunc::Control);
    guarantee(ctrl != NULL, "missing control projection, cannot replace_node() with NULL");
  
!   alock->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/);
    // There are 2 projections from the lock.  The lock node will
    // be deleted when its last use is subsumed below.
    assert(alock->outcnt() == 2 &&
!          _callprojs.fallthrough_proj != NULL &&
!          _callprojs.fallthrough_memproj != NULL,
           "Unexpected projections from Lock/Unlock");
  
!   Node* fallthroughproj = _callprojs.fallthrough_proj;
!   Node* memproj_fallthrough = _callprojs.fallthrough_memproj;
  
    // The memory projection from a lock/unlock is RawMem
    // The input to a Lock is merged memory, so extract its RawMem input
    // (unless the MergeMem has been optimized away.)
    if (alock->is_Lock()) {
      // Seach for MemBarAcquireLock node and delete it also.
      MemBarNode* membar = fallthroughproj->unique_ctrl_out()->as_MemBar();
      assert(membar != NULL && membar->Opcode() == Op_MemBarAcquireLock, "");
      Node* ctrlproj = membar->proj_out(TypeFunc::Control);
      Node* memproj = membar->proj_out(TypeFunc::Memory);
--- 2271,28 ---
  
    Node* mem  = alock->in(TypeFunc::Memory);
    Node* ctrl = alock->in(TypeFunc::Control);
    guarantee(ctrl != NULL, "missing control projection, cannot replace_node() with NULL");
  
!   _callprojs = alock->extract_projections(false /*separate_io_proj*/, false /*do_asserts*/);
    // There are 2 projections from the lock.  The lock node will
    // be deleted when its last use is subsumed below.
    assert(alock->outcnt() == 2 &&
!          _callprojs->fallthrough_proj != NULL &&
!          _callprojs->fallthrough_memproj != NULL,
           "Unexpected projections from Lock/Unlock");
  
!   Node* fallthroughproj = _callprojs->fallthrough_proj;
!   Node* memproj_fallthrough = _callprojs->fallthrough_memproj;
  
    // The memory projection from a lock/unlock is RawMem
    // The input to a Lock is merged memory, so extract its RawMem input
    // (unless the MergeMem has been optimized away.)
    if (alock->is_Lock()) {
+     // Deoptimize and re-execute if object is an inline type
+     inline_type_guard(&ctrl, alock->as_Lock());
+ 
      // Seach for MemBarAcquireLock node and delete it also.
      MemBarNode* membar = fallthroughproj->unique_ctrl_out()->as_MemBar();
      assert(membar != NULL && membar->Opcode() == Op_MemBarAcquireLock, "");
      Node* ctrlproj = membar->proj_out(TypeFunc::Control);
      Node* memproj = membar->proj_out(TypeFunc::Memory);

*** 2171,40 ***
  
    // Optimize test; set region slot 2
    slow_path = opt_bits_test(ctrl, region, 2, flock, 0, 0);
    mem_phi->init_req(2, mem);
  
    // Make slow path call
    CallNode *call = make_slow_call((CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(),
                                    OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path,
                                    obj, box, NULL);
  
!   call->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/);
  
    // Slow path can only throw asynchronous exceptions, which are always
    // de-opted.  So the compiler thinks the slow-call can never throw an
    // exception.  If it DOES throw an exception we would need the debug
    // info removed first (since if it throws there is no monitor).
!   assert(_callprojs.fallthrough_ioproj == NULL && _callprojs.catchall_ioproj == NULL &&
!          _callprojs.catchall_memproj == NULL && _callprojs.catchall_catchproj == NULL, "Unexpected projection from Lock");
  
    // Capture slow path
    // disconnect fall-through projection from call and create a new one
    // hook up users of fall-through projection to region
!   Node *slow_ctrl = _callprojs.fallthrough_proj->clone();
    transform_later(slow_ctrl);
!   _igvn.hash_delete(_callprojs.fallthrough_proj);
!   _callprojs.fallthrough_proj->disconnect_inputs(C);
    region->init_req(1, slow_ctrl);
    // region inputs are now complete
    transform_later(region);
!   _igvn.replace_node(_callprojs.fallthrough_proj, region);
  
    Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory));
    mem_phi->init_req(1, memproj );
    transform_later(mem_phi);
!   _igvn.replace_node(_callprojs.fallthrough_memproj, mem_phi);
  }
  
  //------------------------------expand_unlock_node----------------------
  void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) {
  
--- 2349,43 ---
  
    // Optimize test; set region slot 2
    slow_path = opt_bits_test(ctrl, region, 2, flock, 0, 0);
    mem_phi->init_req(2, mem);
  
+   // Deoptimize and re-execute if object is an inline type
+   inline_type_guard(&slow_path, lock);
+ 
    // Make slow path call
    CallNode *call = make_slow_call((CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(),
                                    OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path,
                                    obj, box, NULL);
  
!   _callprojs = call->extract_projections(false /*separate_io_proj*/, false /*do_asserts*/);
  
    // Slow path can only throw asynchronous exceptions, which are always
    // de-opted.  So the compiler thinks the slow-call can never throw an
    // exception.  If it DOES throw an exception we would need the debug
    // info removed first (since if it throws there is no monitor).
!   assert(_callprojs->fallthrough_ioproj == NULL && _callprojs->catchall_ioproj == NULL &&
!          _callprojs->catchall_memproj == NULL && _callprojs->catchall_catchproj == NULL, "Unexpected projection from Lock");
  
    // Capture slow path
    // disconnect fall-through projection from call and create a new one
    // hook up users of fall-through projection to region
!   Node *slow_ctrl = _callprojs->fallthrough_proj->clone();
    transform_later(slow_ctrl);
!   _igvn.hash_delete(_callprojs->fallthrough_proj);
!   _callprojs->fallthrough_proj->disconnect_inputs(C);
    region->init_req(1, slow_ctrl);
    // region inputs are now complete
    transform_later(region);
!   _igvn.replace_node(_callprojs->fallthrough_proj, region);
  
    Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory));
    mem_phi->init_req(1, memproj );
    transform_later(mem_phi);
!   _igvn.replace_node(_callprojs->fallthrough_memproj, mem_phi);
  }
  
  //------------------------------expand_unlock_node----------------------
  void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) {
  

*** 2233,34 ***
  
    CallNode *call = make_slow_call((CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(),
                                    CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C),
                                    "complete_monitor_unlocking_C", slow_path, obj, box, thread);
  
!   call->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/);
!   assert(_callprojs.fallthrough_ioproj == NULL && _callprojs.catchall_ioproj == NULL &&
!          _callprojs.catchall_memproj == NULL && _callprojs.catchall_catchproj == NULL, "Unexpected projection from Lock");
  
    // No exceptions for unlocking
    // Capture slow path
    // disconnect fall-through projection from call and create a new one
    // hook up users of fall-through projection to region
!   Node *slow_ctrl = _callprojs.fallthrough_proj->clone();
    transform_later(slow_ctrl);
!   _igvn.hash_delete(_callprojs.fallthrough_proj);
!   _callprojs.fallthrough_proj->disconnect_inputs(C);
    region->init_req(1, slow_ctrl);
    // region inputs are now complete
    transform_later(region);
!   _igvn.replace_node(_callprojs.fallthrough_proj, region);
  
    Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory) );
    mem_phi->init_req(1, memproj );
    mem_phi->init_req(2, mem);
    transform_later(mem_phi);
!   _igvn.replace_node(_callprojs.fallthrough_memproj, mem_phi);
  }
  
  void PhaseMacroExpand::expand_subtypecheck_node(SubTypeCheckNode *check) {
    assert(check->in(SubTypeCheckNode::Control) == NULL, "should be pinned");
    Node* bol = check->unique_out();
    Node* obj_or_subklass = check->in(SubTypeCheckNode::ObjOrSubKlass);
    Node* superklass = check->in(SubTypeCheckNode::SuperKlass);
--- 2414,234 ---
  
    CallNode *call = make_slow_call((CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(),
                                    CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C),
                                    "complete_monitor_unlocking_C", slow_path, obj, box, thread);
  
!   _callprojs = call->extract_projections(false /*separate_io_proj*/, false /*do_asserts*/);
!   assert(_callprojs->fallthrough_ioproj == NULL && _callprojs->catchall_ioproj == NULL &&
!          _callprojs->catchall_memproj == NULL && _callprojs->catchall_catchproj == NULL, "Unexpected projection from Lock");
  
    // No exceptions for unlocking
    // Capture slow path
    // disconnect fall-through projection from call and create a new one
    // hook up users of fall-through projection to region
!   Node *slow_ctrl = _callprojs->fallthrough_proj->clone();
    transform_later(slow_ctrl);
!   _igvn.hash_delete(_callprojs->fallthrough_proj);
!   _callprojs->fallthrough_proj->disconnect_inputs(C);
    region->init_req(1, slow_ctrl);
    // region inputs are now complete
    transform_later(region);
!   _igvn.replace_node(_callprojs->fallthrough_proj, region);
  
    Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory) );
    mem_phi->init_req(1, memproj );
    mem_phi->init_req(2, mem);
    transform_later(mem_phi);
!   _igvn.replace_node(_callprojs->fallthrough_memproj, mem_phi);
  }
  
+ // An inline type might be returned from the call but we don't know its
+ // type. Either we get a buffered inline type (and nothing needs to be done)
+ // or one of the inlines being returned is the klass of the inline type
+ // and we need to allocate an inline type instance of that type and
+ // initialize it with other values being returned. In that case, we
+ // first try a fast path allocation and initialize the value with the
+ // inline klass's pack handler or we fall back to a runtime call.
+ void PhaseMacroExpand::expand_mh_intrinsic_return(CallStaticJavaNode* call) {
+   assert(call->method()->is_method_handle_intrinsic(), "must be a method handle intrinsic call");
+   Node* ret = call->proj_out_or_null(TypeFunc::Parms);
+   if (ret == NULL) {
+     return;
+   }
+   const TypeFunc* tf = call->_tf;
+   const TypeTuple* domain = OptoRuntime::store_inline_type_fields_Type()->domain_cc();
+   const TypeFunc* new_tf = TypeFunc::make(tf->domain_sig(), tf->domain_cc(), tf->range_sig(), domain);
+   call->_tf = new_tf;
+   // Make sure the change of type is applied before projections are processed by igvn
+   _igvn.set_type(call, call->Value(&_igvn));
+   _igvn.set_type(ret, ret->Value(&_igvn));
+ 
+   // Before any new projection is added:
+   CallProjections* projs = call->extract_projections(true, true);
+ 
+   // Create temporary hook nodes that will be replaced below.
+   // Add an input to prevent hook nodes from being dead.
+   Node* ctl = new Node(call);
+   Node* mem = new Node(ctl);
+   Node* io = new Node(ctl);
+   Node* ex_ctl = new Node(ctl);
+   Node* ex_mem = new Node(ctl);
+   Node* ex_io = new Node(ctl);
+   Node* res = new Node(ctl);
+ 
+   // Allocate a new buffered inline type only if a new one is not returned
+   Node* cast = transform_later(new CastP2XNode(ctl, res));
+   Node* mask = MakeConX(0x1);
+   Node* masked = transform_later(new AndXNode(cast, mask));
+   Node* cmp = transform_later(new CmpXNode(masked, mask));
+   Node* bol = transform_later(new BoolNode(cmp, BoolTest::eq));
+   IfNode* allocation_iff = new IfNode(ctl, bol, PROB_MAX, COUNT_UNKNOWN);
+   transform_later(allocation_iff);
+   Node* allocation_ctl = transform_later(new IfTrueNode(allocation_iff));
+   Node* no_allocation_ctl = transform_later(new IfFalseNode(allocation_iff));
+   Node* no_allocation_res = transform_later(new CheckCastPPNode(no_allocation_ctl, res, TypeInstPtr::BOTTOM));
+ 
+   // Try to allocate a new buffered inline instance either from TLAB or eden space
+   Node* needgc_ctrl = NULL; // needgc means slowcase, i.e. allocation failed
+   CallLeafNoFPNode* handler_call;
+   const bool alloc_in_place = (UseTLAB || Universe::heap()->supports_inline_contig_alloc());
+   if (alloc_in_place) {
+     Node* fast_oop_ctrl = NULL;
+     Node* fast_oop_rawmem = NULL;
+     Node* mask2 = MakeConX(-2);
+     Node* masked2 = transform_later(new AndXNode(cast, mask2));
+     Node* rawklassptr = transform_later(new CastX2PNode(masked2));
+     Node* klass_node = transform_later(new CheckCastPPNode(allocation_ctl, rawklassptr, TypeInstKlassPtr::OBJECT_OR_NULL));
+     Node* layout_val = make_load(NULL, mem, klass_node, in_bytes(Klass::layout_helper_offset()), TypeInt::INT, T_INT);
+     Node* size_in_bytes = ConvI2X(layout_val);
+     BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+     Node* fast_oop = bs->obj_allocate(this, mem, allocation_ctl, size_in_bytes, io, needgc_ctrl,
+                                       fast_oop_ctrl, fast_oop_rawmem,
+                                       AllocateInstancePrefetchLines);
+     // Allocation succeed, initialize buffered inline instance header firstly,
+     // and then initialize its fields with an inline class specific handler
+     Node* mark_node = makecon(TypeRawPtr::make((address)markWord::inline_type_prototype().value()));
+     fast_oop_rawmem = make_store(fast_oop_ctrl, fast_oop_rawmem, fast_oop, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
+     fast_oop_rawmem = make_store(fast_oop_ctrl, fast_oop_rawmem, fast_oop, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
+     if (UseCompressedClassPointers) {
+       fast_oop_rawmem = make_store(fast_oop_ctrl, fast_oop_rawmem, fast_oop, oopDesc::klass_gap_offset_in_bytes(), intcon(0), T_INT);
+     }
+     Node* fixed_block  = make_load(fast_oop_ctrl, fast_oop_rawmem, klass_node, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
+     Node* pack_handler = make_load(fast_oop_ctrl, fast_oop_rawmem, fixed_block, in_bytes(InlineKlass::pack_handler_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
+     handler_call = new CallLeafNoFPNode(OptoRuntime::pack_inline_type_Type(),
+                                         NULL,
+                                         "pack handler",
+                                         TypeRawPtr::BOTTOM);
+     handler_call->init_req(TypeFunc::Control, fast_oop_ctrl);
+     handler_call->init_req(TypeFunc::Memory, fast_oop_rawmem);
+     handler_call->init_req(TypeFunc::I_O, top());
+     handler_call->init_req(TypeFunc::FramePtr, call->in(TypeFunc::FramePtr));
+     handler_call->init_req(TypeFunc::ReturnAdr, top());
+     handler_call->init_req(TypeFunc::Parms, pack_handler);
+     handler_call->init_req(TypeFunc::Parms+1, fast_oop);
+   } else {
+     needgc_ctrl = allocation_ctl;
+   }
+ 
+   // Allocation failed, fall back to a runtime call
+   CallStaticJavaNode* slow_call = new CallStaticJavaNode(OptoRuntime::store_inline_type_fields_Type(),
+                                                          StubRoutines::store_inline_type_fields_to_buf(),
+                                                          "store_inline_type_fields",
+                                                          TypePtr::BOTTOM);
+   slow_call->init_req(TypeFunc::Control, needgc_ctrl);
+   slow_call->init_req(TypeFunc::Memory, mem);
+   slow_call->init_req(TypeFunc::I_O, io);
+   slow_call->init_req(TypeFunc::FramePtr, call->in(TypeFunc::FramePtr));
+   slow_call->init_req(TypeFunc::ReturnAdr, call->in(TypeFunc::ReturnAdr));
+   slow_call->init_req(TypeFunc::Parms, res);
+ 
+   Node* slow_ctl = transform_later(new ProjNode(slow_call, TypeFunc::Control));
+   Node* slow_mem = transform_later(new ProjNode(slow_call, TypeFunc::Memory));
+   Node* slow_io = transform_later(new ProjNode(slow_call, TypeFunc::I_O));
+   Node* slow_res = transform_later(new ProjNode(slow_call, TypeFunc::Parms));
+   Node* slow_catc = transform_later(new CatchNode(slow_ctl, slow_io, 2));
+   Node* slow_norm = transform_later(new CatchProjNode(slow_catc, CatchProjNode::fall_through_index, CatchProjNode::no_handler_bci));
+   Node* slow_excp = transform_later(new CatchProjNode(slow_catc, CatchProjNode::catch_all_index,    CatchProjNode::no_handler_bci));
+ 
+   Node* ex_r = new RegionNode(3);
+   Node* ex_mem_phi = new PhiNode(ex_r, Type::MEMORY, TypePtr::BOTTOM);
+   Node* ex_io_phi = new PhiNode(ex_r, Type::ABIO);
+   ex_r->init_req(1, slow_excp);
+   ex_mem_phi->init_req(1, slow_mem);
+   ex_io_phi->init_req(1, slow_io);
+   ex_r->init_req(2, ex_ctl);
+   ex_mem_phi->init_req(2, ex_mem);
+   ex_io_phi->init_req(2, ex_io);
+   transform_later(ex_r);
+   transform_later(ex_mem_phi);
+   transform_later(ex_io_phi);
+ 
+   // We don't know how many values are returned. This assumes the
+   // worst case, that all available registers are used.
+   for (uint i = TypeFunc::Parms+1; i < domain->cnt(); i++) {
+     if (domain->field_at(i) == Type::HALF) {
+       slow_call->init_req(i, top());
+       if (alloc_in_place) {
+         handler_call->init_req(i+1, top());
+       }
+       continue;
+     }
+     Node* proj = transform_later(new ProjNode(call, i));
+     slow_call->init_req(i, proj);
+     if (alloc_in_place) {
+       handler_call->init_req(i+1, proj);
+     }
+   }
+   // We can safepoint at that new call
+   slow_call->copy_call_debug_info(&_igvn, call);
+   transform_later(slow_call);
+   if (alloc_in_place) {
+     transform_later(handler_call);
+   }
+ 
+   Node* fast_ctl = NULL;
+   Node* fast_res = NULL;
+   MergeMemNode* fast_mem = NULL;
+   if (alloc_in_place) {
+     fast_ctl = transform_later(new ProjNode(handler_call, TypeFunc::Control));
+     Node* rawmem = transform_later(new ProjNode(handler_call, TypeFunc::Memory));
+     fast_res = transform_later(new ProjNode(handler_call, TypeFunc::Parms));
+     fast_mem = MergeMemNode::make(mem);
+     fast_mem->set_memory_at(Compile::AliasIdxRaw, rawmem);
+     transform_later(fast_mem);
+   }
+ 
+   Node* r = new RegionNode(alloc_in_place ? 4 : 3);
+   Node* mem_phi = new PhiNode(r, Type::MEMORY, TypePtr::BOTTOM);
+   Node* io_phi = new PhiNode(r, Type::ABIO);
+   Node* res_phi = new PhiNode(r, TypeInstPtr::BOTTOM);
+   r->init_req(1, no_allocation_ctl);
+   mem_phi->init_req(1, mem);
+   io_phi->init_req(1, io);
+   res_phi->init_req(1, no_allocation_res);
+   r->init_req(2, slow_norm);
+   mem_phi->init_req(2, slow_mem);
+   io_phi->init_req(2, slow_io);
+   res_phi->init_req(2, slow_res);
+   if (alloc_in_place) {
+     r->init_req(3, fast_ctl);
+     mem_phi->init_req(3, fast_mem);
+     io_phi->init_req(3, io);
+     res_phi->init_req(3, fast_res);
+   }
+   transform_later(r);
+   transform_later(mem_phi);
+   transform_later(io_phi);
+   transform_later(res_phi);
+ 
+   assert(projs->nb_resproj == 1, "unexpected number of results");
+   _igvn.replace_in_uses(projs->fallthrough_catchproj, r);
+   _igvn.replace_in_uses(projs->fallthrough_memproj, mem_phi);
+   _igvn.replace_in_uses(projs->fallthrough_ioproj, io_phi);
+   _igvn.replace_in_uses(projs->resproj[0], res_phi);
+   _igvn.replace_in_uses(projs->catchall_catchproj, ex_r);
+   _igvn.replace_in_uses(projs->catchall_memproj, ex_mem_phi);
+   _igvn.replace_in_uses(projs->catchall_ioproj, ex_io_phi);
+   // The CatchNode should not use the ex_io_phi. Re-connect it to the catchall_ioproj.
+   Node* cn = projs->fallthrough_catchproj->in(0);
+   _igvn.replace_input_of(cn, 1, projs->catchall_ioproj);
+ 
+   _igvn.replace_node(ctl, projs->fallthrough_catchproj);
+   _igvn.replace_node(mem, projs->fallthrough_memproj);
+   _igvn.replace_node(io, projs->fallthrough_ioproj);
+   _igvn.replace_node(res, projs->resproj[0]);
+   _igvn.replace_node(ex_ctl, projs->catchall_catchproj);
+   _igvn.replace_node(ex_mem, projs->catchall_memproj);
+   _igvn.replace_node(ex_io, projs->catchall_ioproj);
+  }
+ 
  void PhaseMacroExpand::expand_subtypecheck_node(SubTypeCheckNode *check) {
    assert(check->in(SubTypeCheckNode::Control) == NULL, "should be pinned");
    Node* bol = check->unique_out();
    Node* obj_or_subklass = check->in(SubTypeCheckNode::ObjOrSubKlass);
    Node* superklass = check->in(SubTypeCheckNode::SuperKlass);

*** 2282,11 ***
      Node* subklass = NULL;
      if (_igvn.type(obj_or_subklass)->isa_klassptr()) {
        subklass = obj_or_subklass;
      } else {
        Node* k_adr = basic_plus_adr(obj_or_subklass, oopDesc::klass_offset_in_bytes());
!       subklass = _igvn.transform(LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), k_adr, TypeInstPtr::KLASS));
      }
  
      Node* not_subtype_ctrl = Phase::gen_subtype_check(subklass, superklass, &ctrl, NULL, _igvn);
  
      _igvn.replace_input_of(iff, 0, C->top());
--- 2663,11 ---
      Node* subklass = NULL;
      if (_igvn.type(obj_or_subklass)->isa_klassptr()) {
        subklass = obj_or_subklass;
      } else {
        Node* k_adr = basic_plus_adr(obj_or_subklass, oopDesc::klass_offset_in_bytes());
!       subklass = _igvn.transform(LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), k_adr, TypeInstPtr::KLASS, TypeInstKlassPtr::OBJECT));
      }
  
      Node* not_subtype_ctrl = Phase::gen_subtype_check(subklass, superklass, &ctrl, NULL, _igvn);
  
      _igvn.replace_input_of(iff, 0, C->top());

*** 2294,10 ***
--- 2675,106 ---
      _igvn.replace_node(iffalse, ctrl);
    }
    _igvn.replace_node(check, C->top());
  }
  
+ // FlatArrayCheckNode (array1 array2 ...) is expanded into:
+ //
+ // long mark = array1.mark | array2.mark | ...;
+ // long locked_bit = markWord::unlocked_value & array1.mark & array2.mark & ...;
+ // if (locked_bit == 0) {
+ //   // One array is locked, load prototype header from the klass
+ //   mark = array1.klass.proto | array2.klass.proto | ...
+ // }
+ // if ((mark & markWord::flat_array_bit_in_place) == 0) {
+ //    ...
+ // }
+ void PhaseMacroExpand::expand_flatarraycheck_node(FlatArrayCheckNode* check) {
+   if (UseArrayMarkWordCheck) {
+     Node* mark = MakeConX(0);
+     Node* locked_bit = MakeConX(markWord::unlocked_value);
+     Node* mem = check->in(FlatArrayCheckNode::Memory);
+     for (uint i = FlatArrayCheckNode::Array; i < check->req(); ++i) {
+       Node* ary = check->in(i);
+       if (ary->is_top()) continue;
+       const TypeAryPtr* t = _igvn.type(ary)->isa_aryptr();
+       assert(!t->is_flat() && !t->is_not_flat(), "Should have been optimized out");
+       Node* mark_adr = basic_plus_adr(ary, oopDesc::mark_offset_in_bytes());
+       Node* mark_load = _igvn.transform(LoadNode::make(_igvn, NULL, mem, mark_adr, mark_adr->bottom_type()->is_ptr(), TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
+       mark = _igvn.transform(new OrXNode(mark, mark_load));
+       locked_bit = _igvn.transform(new AndXNode(locked_bit, mark_load));
+     }
+     assert(!mark->is_Con(), "Should have been optimized out");
+     Node* cmp = _igvn.transform(new CmpXNode(locked_bit, MakeConX(0)));
+     Node* is_unlocked = _igvn.transform(new BoolNode(cmp, BoolTest::ne));
+ 
+     // BoolNode might be shared, replace each if user
+     Node* old_bol = check->unique_out();
+     assert(old_bol->is_Bool() && old_bol->as_Bool()->_test._test == BoolTest::ne, "unexpected condition");
+     for (DUIterator_Last imin, i = old_bol->last_outs(imin); i >= imin; --i) {
+       IfNode* old_iff = old_bol->last_out(i)->as_If();
+       Node* ctrl = old_iff->in(0);
+       RegionNode* region = new RegionNode(3);
+       Node* mark_phi = new PhiNode(region, TypeX_X);
+ 
+       // Check if array is unlocked
+       IfNode* iff = _igvn.transform(new IfNode(ctrl, is_unlocked, PROB_MAX, COUNT_UNKNOWN))->as_If();
+ 
+       // Unlocked: Use bits from mark word
+       region->init_req(1, _igvn.transform(new IfTrueNode(iff)));
+       mark_phi->init_req(1, mark);
+ 
+       // Locked: Load prototype header from klass
+       ctrl = _igvn.transform(new IfFalseNode(iff));
+       Node* proto = MakeConX(0);
+       for (uint i = FlatArrayCheckNode::Array; i < check->req(); ++i) {
+         Node* ary = check->in(i);
+         if (ary->is_top()) continue;
+         // Make loads control dependent to make sure they are only executed if array is locked
+         Node* klass_adr = basic_plus_adr(ary, oopDesc::klass_offset_in_bytes());
+         Node* klass = _igvn.transform(LoadKlassNode::make(_igvn, ctrl, C->immutable_memory(), klass_adr, TypeInstPtr::KLASS, TypeInstKlassPtr::OBJECT));
+         Node* proto_adr = basic_plus_adr(klass, in_bytes(Klass::prototype_header_offset()));
+         Node* proto_load = _igvn.transform(LoadNode::make(_igvn, ctrl, C->immutable_memory(), proto_adr, proto_adr->bottom_type()->is_ptr(), TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
+         proto = _igvn.transform(new OrXNode(proto, proto_load));
+       }
+       region->init_req(2, ctrl);
+       mark_phi->init_req(2, proto);
+ 
+       // Check if flat array bits are set
+       Node* mask = MakeConX(markWord::flat_array_bit_in_place);
+       Node* masked = _igvn.transform(new AndXNode(_igvn.transform(mark_phi), mask));
+       cmp = _igvn.transform(new CmpXNode(masked, MakeConX(0)));
+       Node* is_not_flat = _igvn.transform(new BoolNode(cmp, BoolTest::eq));
+ 
+       ctrl = _igvn.transform(region);
+       iff = _igvn.transform(new IfNode(ctrl, is_not_flat, PROB_MAX, COUNT_UNKNOWN))->as_If();
+       _igvn.replace_node(old_iff, iff);
+     }
+     _igvn.replace_node(check, C->top());
+   } else {
+     // Fall back to layout helper check
+     Node* lhs = intcon(0);
+     for (uint i = FlatArrayCheckNode::Array; i < check->req(); ++i) {
+       Node* ary = check->in(i);
+       if (ary->is_top()) continue;
+       const TypeAryPtr* t = _igvn.type(ary)->isa_aryptr();
+       assert(!t->is_flat() && !t->is_not_flat(), "Should have been optimized out");
+       Node* klass_adr = basic_plus_adr(ary, oopDesc::klass_offset_in_bytes());
+       Node* klass = transform_later(LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), klass_adr, TypeInstPtr::KLASS, TypeInstKlassPtr::OBJECT));
+       Node* lh_addr = basic_plus_adr(klass, in_bytes(Klass::layout_helper_offset()));
+       Node* lh_val = _igvn.transform(LoadNode::make(_igvn, NULL, C->immutable_memory(), lh_addr, lh_addr->bottom_type()->is_ptr(), TypeInt::INT, T_INT, MemNode::unordered));
+       lhs = _igvn.transform(new OrINode(lhs, lh_val));
+     }
+     Node* masked = transform_later(new AndINode(lhs, intcon(Klass::_lh_array_tag_vt_value_bit_inplace)));
+     Node* cmp = transform_later(new CmpINode(masked, intcon(0)));
+     Node* bol = transform_later(new BoolNode(cmp, BoolTest::eq));
+     Node* old_bol = check->unique_out();
+     _igvn.replace_node(old_bol, bol);
+     _igvn.replace_node(check, C->top());
+   }
+ }
+ 
  //---------------------------eliminate_macro_nodes----------------------
  // Eliminate scalar replaced allocations and associated locks.
  void PhaseMacroExpand::eliminate_macro_nodes() {
    if (C->macro_count() == 0)
      return;

*** 2343,13 ***
        switch (n->class_id()) {
        case Node::Class_Allocate:
        case Node::Class_AllocateArray:
          success = eliminate_allocate_node(n->as_Allocate());
          break;
!       case Node::Class_CallStaticJava:
!         success = eliminate_boxing_node(n->as_CallStaticJava());
          break;
        case Node::Class_Lock:
        case Node::Class_Unlock:
          assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
          _has_locks = true;
          break;
--- 2820,17 ---
        switch (n->class_id()) {
        case Node::Class_Allocate:
        case Node::Class_AllocateArray:
          success = eliminate_allocate_node(n->as_Allocate());
          break;
!       case Node::Class_CallStaticJava: {
!         CallStaticJavaNode* call = n->as_CallStaticJava();
+         if (!call->method()->is_method_handle_intrinsic()) {
+           success = eliminate_boxing_node(n->as_CallStaticJava());
+         }
          break;
+       }
        case Node::Class_Lock:
        case Node::Class_Unlock:
          assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
          _has_locks = true;
          break;

*** 2359,10 ***
--- 2840,12 ---
          break;
        case Node::Class_SubTypeCheck:
          break;
        case Node::Class_Opaque1:
          break;
+       case Node::Class_FlatArrayCheck:
+         break;
        default:
          assert(n->Opcode() == Op_LoopLimit ||
                 n->Opcode() == Op_Opaque2   ||
                 n->Opcode() == Op_Opaque3   ||
                 BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n),

*** 2393,14 ***
          // Remove it from macro list and put on IGVN worklist to optimize.
          C->remove_macro_node(n);
          _igvn._worklist.push(n);
          success = true;
        } else if (n->Opcode() == Op_CallStaticJava) {
!         // Remove it from macro list and put on IGVN worklist to optimize.
!         C->remove_macro_node(n);
!         _igvn._worklist.push(n);
!         success = true;
        } else if (n->is_Opaque1() || n->Opcode() == Op_Opaque2) {
          _igvn.replace_node(n, n->in(1));
          success = true;
  #if INCLUDE_RTM_OPT
        } else if ((n->Opcode() == Op_Opaque3) && ((Opaque3Node*)n)->rtm_opt()) {
--- 2876,17 ---
          // Remove it from macro list and put on IGVN worklist to optimize.
          C->remove_macro_node(n);
          _igvn._worklist.push(n);
          success = true;
        } else if (n->Opcode() == Op_CallStaticJava) {
!         CallStaticJavaNode* call = n->as_CallStaticJava();
!         if (!call->method()->is_method_handle_intrinsic()) {
!           // Remove it from macro list and put on IGVN worklist to optimize.
!           C->remove_macro_node(n);
+           _igvn._worklist.push(n);
+           success = true;
+         }
        } else if (n->is_Opaque1() || n->Opcode() == Op_Opaque2) {
          _igvn.replace_node(n, n->in(1));
          success = true;
  #if INCLUDE_RTM_OPT
        } else if ((n->Opcode() == Op_Opaque3) && ((Opaque3Node*)n)->rtm_opt()) {

*** 2486,10 ***
--- 2972,17 ---
        expand_arraycopy_node(n->as_ArrayCopy());
        break;
      case Node::Class_SubTypeCheck:
        expand_subtypecheck_node(n->as_SubTypeCheck());
        break;
+     case Node::Class_CallStaticJava:
+       expand_mh_intrinsic_return(n->as_CallStaticJava());
+       C->remove_macro_node(n);
+       break;
+     case Node::Class_FlatArrayCheck:
+       expand_flatarraycheck_node(n->as_FlatArrayCheck());
+       break;
      default:
        assert(false, "unknown node type in macro list");
      }
      assert(C->macro_count() == (old_macro_count - 1), "expansion must have deleted one node from macro list");
      if (C->failing())  return true;
< prev index next >