diff a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp
--- a/src/hotspot/share/c1/c1_LIRGenerator.cpp
+++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp
@@ -29,10 +29,12 @@
 #include "c1/c1_Instruction.hpp"
 #include "c1/c1_LIRAssembler.hpp"
 #include "c1/c1_LIRGenerator.hpp"
 #include "c1/c1_ValueStack.hpp"
 #include "ci/ciArrayKlass.hpp"
+#include "ci/ciFlatArrayKlass.hpp"
+#include "ci/ciInlineKlass.hpp"
 #include "ci/ciInstance.hpp"
 #include "ci/ciObjArray.hpp"
 #include "ci/ciUtilities.hpp"
 #include "compiler/compilerDefinitions.inline.hpp"
 #include "compiler/compilerOracle.hpp"
@@ -214,10 +216,12 @@
 
   _result = opr;
 }
 
 void LIRItem::load_item() {
+  assert(!_gen->in_conditional_code(), "LIRItem cannot be loaded in conditional code");
+
   if (result()->is_illegal()) {
     // update the items result
     _result = value()->operand();
   }
   if (!result()->is_register()) {
@@ -621,17 +625,18 @@
     default: ShouldNotReachHere();
   }
 }
 
 
-void LIRGenerator::monitor_enter(LIR_Opr object, LIR_Opr lock, LIR_Opr hdr, LIR_Opr scratch, int monitor_no, CodeEmitInfo* info_for_exception, CodeEmitInfo* info) {
+void LIRGenerator::monitor_enter(LIR_Opr object, LIR_Opr lock, LIR_Opr hdr, LIR_Opr scratch, int monitor_no,
+                                 CodeEmitInfo* info_for_exception, CodeEmitInfo* info, CodeStub* throw_ie_stub) {
   if (!GenerateSynchronizationCode) return;
   // for slow path, use debug info for state after successful locking
-  CodeStub* slow_path = new MonitorEnterStub(object, lock, info);
+  CodeStub* slow_path = new MonitorEnterStub(object, lock, info, throw_ie_stub, scratch);
   __ load_stack_address_monitor(monitor_no, lock);
   // for handling NullPointerException, use debug info representing just the lock stack before this monitorenter
-  __ lock_object(hdr, object, lock, scratch, slow_path, info_for_exception);
+  __ lock_object(hdr, object, lock, scratch, slow_path, info_for_exception, throw_ie_stub);
 }
 
 
 void LIRGenerator::monitor_exit(LIR_Opr object, LIR_Opr lock, LIR_Opr new_hdr, LIR_Opr scratch, int monitor_no) {
   if (!GenerateSynchronizationCode) return;
@@ -651,14 +656,19 @@
     tty->print_cr("   ###class not resolved at new bci %d", new_instance->printable_bci());
   }
 }
 #endif
 
-void LIRGenerator::new_instance(LIR_Opr dst, ciInstanceKlass* klass, bool is_unresolved, LIR_Opr scratch1, LIR_Opr scratch2, LIR_Opr scratch3, LIR_Opr scratch4, LIR_Opr klass_reg, CodeEmitInfo* info) {
-  klass2reg_with_patching(klass_reg, klass, info, is_unresolved);
-  // If klass is not loaded we do not know if the klass has finalizers:
-  if (UseFastNewInstance && klass->is_loaded()
+void LIRGenerator::new_instance(LIR_Opr dst, ciInstanceKlass* klass, bool is_unresolved, bool allow_inline, LIR_Opr scratch1, LIR_Opr scratch2, LIR_Opr scratch3, LIR_Opr scratch4, LIR_Opr klass_reg, CodeEmitInfo* info) {
+  if (allow_inline) {
+    assert(!is_unresolved && klass->is_loaded(), "inline type klass should be resolved");
+    __ metadata2reg(klass->constant_encoding(), klass_reg);
+  } else {
+    klass2reg_with_patching(klass_reg, klass, info, is_unresolved);
+  }
+  // If klass is not loaded we do not know if the klass has finalizers or is an unexpected inline klass
+  if (UseFastNewInstance && klass->is_loaded() && (allow_inline || !klass->is_inlinetype())
       && !Klass::layout_helper_needs_slow_path(klass->layout_helper())) {
 
     Runtime1::StubID stub_id = klass->is_initialized() ? Runtime1::fast_new_instance_id : Runtime1::fast_new_instance_init_check_id;
 
     CodeStub* slow_path = new NewInstanceStub(klass_reg, dst, klass, info, stub_id);
@@ -669,11 +679,11 @@
     const int instance_size = align_object_size(klass->size_helper());
     __ allocate_object(dst, scratch1, scratch2, scratch3, scratch4,
                        oopDesc::header_size(), instance_size, klass_reg, !klass->is_initialized(), slow_path);
   } else {
     CodeStub* slow_path = new NewInstanceStub(klass_reg, dst, klass, info, Runtime1::new_instance_id);
-    __ branch(lir_cond_always, slow_path);
+    __ jump(slow_path);
     __ branch_destination(slow_path->continuation());
   }
 }
 
 
@@ -769,10 +779,20 @@
 
   // if a probable array type has been identified, figure out if any
   // of the required checks for a fast case can be elided.
   int flags = LIR_OpArrayCopy::all_flags;
 
+  if (!src->is_loaded_flat_array() && !dst->is_loaded_flat_array()) {
+    flags &= ~LIR_OpArrayCopy::always_slow_path;
+  }
+  if (!src->maybe_flat_array()) {
+    flags &= ~LIR_OpArrayCopy::src_inlinetype_check;
+  }
+  if (!dst->maybe_flat_array() && !dst->maybe_null_free_array()) {
+    flags &= ~LIR_OpArrayCopy::dst_inlinetype_check;
+  }
+
   if (!src_objarray)
     flags &= ~LIR_OpArrayCopy::src_objarray;
   if (!dst_objarray)
     flags &= ~LIR_OpArrayCopy::dst_objarray;
 
@@ -1322,24 +1342,39 @@
   // much less risk of confusion for C1 register allocator. The choice of the universe
   // object here is correct as long as it returns the same modifiers we would expect
   // from the primitive class itself. See spec for Class.getModifiers that provides
   // the typed array klasses with similar modifiers as their component types.
 
+  // Valhalla update: the code is now a bit convuloted because arrays and primitive
+  // classes don't have the same modifiers set anymore, but we cannot introduce
+  // branches in LIR generation (JDK-8211231). So, the first part of the code remains
+  // identical, using the byteArrayKlass object to avoid a NPE when accessing the
+  // modifiers. But then the code also prepares the correct modifiers set for
+  // primitive classes, and there's a second conditional move to put the right
+  // value into result.
+
+
   Klass* univ_klass = Universe::byteArrayKlass();
-  assert(univ_klass->modifier_flags() == (JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC), "Sanity");
+  assert(univ_klass->modifier_flags() == (JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC
+                                          | (Arguments::enable_preview() ? JVM_ACC_IDENTITY : 0)), "Sanity");
   LIR_Opr prim_klass = LIR_OprFact::metadataConst(univ_klass);
 
   LIR_Opr recv_klass = new_register(T_METADATA);
   __ move(new LIR_Address(receiver.result(), java_lang_Class::klass_offset(), T_ADDRESS), recv_klass, info);
 
   // Check if this is a Java mirror of primitive type, and select the appropriate klass.
   LIR_Opr klass = new_register(T_METADATA);
   __ cmp(lir_cond_equal, recv_klass, LIR_OprFact::metadataConst(nullptr));
   __ cmove(lir_cond_equal, prim_klass, recv_klass, klass, T_ADDRESS);
+  LIR_Opr klass_modifiers = new_register(T_INT);
+  __ move(new LIR_Address(klass, in_bytes(Klass::modifier_flags_offset()), T_INT), klass_modifiers);
+
+  LIR_Opr prim_modifiers = load_immediate(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC, T_INT);
+
+  __ cmp(lir_cond_equal, recv_klass, LIR_OprFact::metadataConst(0));
+  __ cmove(lir_cond_equal, prim_modifiers, klass_modifiers, result, T_INT);
 
-  // Get the answer.
-  __ move(new LIR_Address(klass, in_bytes(Klass::modifier_flags_offset()), T_INT), result);
 }
 
 void LIRGenerator::do_getObjectSize(Intrinsic* x) {
   assert(x->number_of_arguments() == 3, "wrong type");
   LIR_Opr result_reg = rlock_result(x);
@@ -1568,15 +1603,23 @@
     }
   }
 
   LIR_Opr result = new_register(t);
   __ move((LIR_Opr)c, result);
-  _constants.append(c);
-  _reg_for_constants.append(result);
+  if (!in_conditional_code()) {
+    _constants.append(c);
+    _reg_for_constants.append(result);
+  }
   return result;
 }
 
+void LIRGenerator::set_in_conditional_code(bool v) {
+  assert(v != _in_conditional_code, "must change state");
+  _in_conditional_code = v;
+}
+
+
 //------------------------field access--------------------------------------
 
 void LIRGenerator::do_CompareAndSwap(Intrinsic* x, ValueType* type) {
   assert(x->number_of_arguments() == 4, "wrong type");
   LIRItem obj   (x->argument_at(0), this);  // object
@@ -1685,18 +1728,193 @@
 
   access_store_at(decorators, field_type, object, LIR_OprFact::intConst(x->offset()),
                   value.result(), info != nullptr ? new CodeEmitInfo(info) : nullptr, info);
 }
 
+// FIXME -- I can't find any other way to pass an address to access_load_at().
+class TempResolvedAddress: public Instruction {
+ public:
+  TempResolvedAddress(ValueType* type, LIR_Opr addr) : Instruction(type) {
+    set_operand(addr);
+  }
+  virtual void input_values_do(ValueVisitor*) {}
+  virtual void visit(InstructionVisitor* v)   {}
+  virtual const char* name() const  { return "TempResolvedAddress"; }
+};
+
+LIR_Opr LIRGenerator::get_and_load_element_address(LIRItem& array, LIRItem& index) {
+  ciType* array_type = array.value()->declared_type();
+  ciFlatArrayKlass* flat_array_klass = array_type->as_flat_array_klass();
+  assert(flat_array_klass->is_loaded(), "must be");
+
+  int array_header_size = flat_array_klass->array_header_in_bytes();
+  int shift = flat_array_klass->log2_element_size();
+
+#ifndef _LP64
+  LIR_Opr index_op = new_register(T_INT);
+  // FIXME -- on 32-bit, the shift below can overflow, so we need to check that
+  // the top (shift+1) bits of index_op must be zero, or
+  // else throw ArrayIndexOutOfBoundsException
+  if (index.result()->is_constant()) {
+    jint const_index = index.result()->as_jint();
+    __ move(LIR_OprFact::intConst(const_index << shift), index_op);
+  } else {
+    __ shift_left(index_op, shift, index.result());
+  }
+#else
+  LIR_Opr index_op = new_register(T_LONG);
+  if (index.result()->is_constant()) {
+    jint const_index = index.result()->as_jint();
+    __ move(LIR_OprFact::longConst(const_index << shift), index_op);
+  } else {
+    __ convert(Bytecodes::_i2l, index.result(), index_op);
+    // Need to shift manually, as LIR_Address can scale only up to 3.
+    __ shift_left(index_op, shift, index_op);
+  }
+#endif
+
+  LIR_Opr elm_op = new_pointer_register();
+  LIR_Address* elm_address = generate_address(array.result(), index_op, 0, array_header_size, T_ADDRESS);
+  __ leal(LIR_OprFact::address(elm_address), elm_op);
+  return elm_op;
+}
+
+void LIRGenerator::access_sub_element(LIRItem& array, LIRItem& index, LIR_Opr& result, ciField* field, int sub_offset) {
+  assert(field != nullptr, "Need a subelement type specified");
+
+  // Find the starting address of the source (inside the array)
+  LIR_Opr elm_op = get_and_load_element_address(array, index);
+
+  BasicType subelt_type = field->type()->basic_type();
+  TempResolvedAddress* elm_resolved_addr = new TempResolvedAddress(as_ValueType(subelt_type), elm_op);
+  LIRItem elm_item(elm_resolved_addr, this);
+
+  DecoratorSet decorators = IN_HEAP;
+  access_load_at(decorators, subelt_type,
+                     elm_item, LIR_OprFact::intConst(sub_offset), result,
+                     nullptr, nullptr);
+
+  if (field->is_null_free()) {
+    assert(field->type()->is_loaded(), "Must be");
+    assert(field->type()->is_inlinetype(), "Must be if loaded");
+    assert(field->type()->as_inline_klass()->is_initialized(), "Must be");
+    LabelObj* L_end = new LabelObj();
+    __ cmp(lir_cond_notEqual, result, LIR_OprFact::oopConst(nullptr));
+    __ branch(lir_cond_notEqual, L_end->label());
+    set_in_conditional_code(true);
+    Constant* default_value = new Constant(new InstanceConstant(field->type()->as_inline_klass()->default_instance()));
+    if (default_value->is_pinned()) {
+      __ move(LIR_OprFact::value_type(default_value->type()), result);
+    } else {
+      __ move(load_constant(default_value), result);
+    }
+    __ branch_destination(L_end->label());
+    set_in_conditional_code(false);
+  }
+}
+
+void LIRGenerator::access_flat_array(bool is_load, LIRItem& array, LIRItem& index, LIRItem& obj_item,
+                                          ciField* field, int sub_offset) {
+  assert(sub_offset == 0 || field != nullptr, "Sanity check");
+
+  // Find the starting address of the source (inside the array)
+  LIR_Opr elm_op = get_and_load_element_address(array, index);
+
+  ciInlineKlass* elem_klass = nullptr;
+  if (field != nullptr) {
+    elem_klass = field->type()->as_inline_klass();
+  } else {
+    elem_klass = array.value()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass();
+  }
+  for (int i = 0; i < elem_klass->nof_nonstatic_fields(); i++) {
+    ciField* inner_field = elem_klass->nonstatic_field_at(i);
+    assert(!inner_field->is_flat(), "flat fields must have been expanded");
+    int obj_offset = inner_field->offset_in_bytes();
+    int elm_offset = obj_offset - elem_klass->first_field_offset() + sub_offset; // object header is not stored in array.
+    BasicType field_type = inner_field->type()->basic_type();
+
+    // Types which are smaller than int are still passed in an int register.
+    BasicType reg_type = field_type;
+    switch (reg_type) {
+    case T_BYTE:
+    case T_BOOLEAN:
+    case T_SHORT:
+    case T_CHAR:
+      reg_type = T_INT;
+      break;
+    default:
+      break;
+    }
+
+    LIR_Opr temp = new_register(reg_type);
+    TempResolvedAddress* elm_resolved_addr = new TempResolvedAddress(as_ValueType(field_type), elm_op);
+    LIRItem elm_item(elm_resolved_addr, this);
+
+    DecoratorSet decorators = IN_HEAP;
+    if (is_load) {
+      access_load_at(decorators, field_type,
+                     elm_item, LIR_OprFact::intConst(elm_offset), temp,
+                     nullptr, nullptr);
+      access_store_at(decorators, field_type,
+                      obj_item, LIR_OprFact::intConst(obj_offset), temp,
+                      nullptr, nullptr);
+    } else {
+      access_load_at(decorators, field_type,
+                     obj_item, LIR_OprFact::intConst(obj_offset), temp,
+                     nullptr, nullptr);
+      access_store_at(decorators, field_type,
+                      elm_item, LIR_OprFact::intConst(elm_offset), temp,
+                      nullptr, nullptr);
+    }
+  }
+}
+
+void LIRGenerator::check_flat_array(LIR_Opr array, LIR_Opr value, CodeStub* slow_path) {
+  LIR_Opr tmp = new_register(T_METADATA);
+  __ check_flat_array(array, value, tmp, slow_path);
+}
+
+void LIRGenerator::check_null_free_array(LIRItem& array, LIRItem& value, CodeEmitInfo* info) {
+  LabelObj* L_end = new LabelObj();
+  LIR_Opr tmp = new_register(T_METADATA);
+  __ check_null_free_array(array.result(), tmp);
+  __ branch(lir_cond_equal, L_end->label());
+  __ null_check(value.result(), info);
+  __ branch_destination(L_end->label());
+}
+
+bool LIRGenerator::needs_flat_array_store_check(StoreIndexed* x) {
+  if (x->elt_type() == T_OBJECT && x->array()->maybe_flat_array()) {
+    ciType* type = x->value()->declared_type();
+    if (type != nullptr && type->is_klass()) {
+      ciKlass* klass = type->as_klass();
+      if (!klass->can_be_inline_klass() || (klass->is_inlinetype() && !klass->as_inline_klass()->flat_in_array())) {
+        // This is known to be a non-flat object. If the array is a flat array,
+        // it will be caught by the code generated by array_store_check().
+        return false;
+      }
+    }
+    // We're not 100% sure, so let's do the flat_array_store_check.
+    return true;
+  }
+  return false;
+}
+
+bool LIRGenerator::needs_null_free_array_store_check(StoreIndexed* x) {
+  return x->elt_type() == T_OBJECT && x->array()->maybe_null_free_array();
+}
+
 void LIRGenerator::do_StoreIndexed(StoreIndexed* x) {
   assert(x->is_pinned(),"");
+  assert(x->elt_type() != T_ARRAY, "never used");
+  bool is_loaded_flat_array = x->array()->is_loaded_flat_array();
   bool needs_range_check = x->compute_needs_range_check();
   bool use_length = x->length() != nullptr;
   bool obj_store = is_reference_type(x->elt_type());
-  bool needs_store_check = obj_store && (x->value()->as_Constant() == nullptr ||
-                                         !get_jobject_constant(x->value())->is_null_object() ||
-                                         x->should_profile());
+  bool needs_store_check = obj_store && !(is_loaded_flat_array && x->is_exact_flat_array_store()) &&
+                                        (x->value()->as_Constant() == nullptr ||
+                                         !get_jobject_constant(x->value())->is_null_object());
 
   LIRItem array(x->array(), this);
   LIRItem index(x->index(), this);
   LIRItem value(x->value(), this);
   LIRItem length(this);
@@ -1705,13 +1923,14 @@
   index.load_nonconstant();
 
   if (use_length && needs_range_check) {
     length.set_instruction(x->length());
     length.load_item();
-
   }
-  if (needs_store_check || x->check_boolean()) {
+
+  if (needs_store_check || x->check_boolean()
+      || is_loaded_flat_array || needs_flat_array_store_check(x) || needs_null_free_array_store_check(x)) {
     value.load_item();
   } else {
     value.load_for_store(x->elt_type());
   }
 
@@ -1735,22 +1954,69 @@
       // range_check also does the null check
       null_check_info = nullptr;
     }
   }
 
+  if (x->should_profile()) {
+    if (x->array()->is_loaded_flat_array()) {
+      // No need to profile a store to a flat array of known type. This can happen if
+      // the type only became known after optimizations (for example, after the PhiSimplifier).
+      x->set_should_profile(false);
+    } else {
+      int bci = x->profiled_bci();
+      ciMethodData* md = x->profiled_method()->method_data();
+      assert(md != nullptr, "Sanity");
+      ciProfileData* data = md->bci_to_data(bci);
+      assert(data != nullptr && data->is_ArrayStoreData(), "incorrect profiling entry");
+      ciArrayStoreData* store_data = (ciArrayStoreData*)data;
+      profile_array_type(x, md, store_data);
+      assert(store_data->is_ArrayStoreData(), "incorrect profiling entry");
+      if (x->array()->maybe_null_free_array()) {
+        profile_null_free_array(array, md, store_data);
+      }
+    }
+  }
+
   if (GenerateArrayStoreCheck && needs_store_check) {
     CodeEmitInfo* store_check_info = new CodeEmitInfo(range_check_info);
     array_store_check(value.result(), array.result(), store_check_info, x->profiled_method(), x->profiled_bci());
   }
 
-  DecoratorSet decorators = IN_HEAP | IS_ARRAY;
-  if (x->check_boolean()) {
-    decorators |= C1_MASK_BOOLEAN;
-  }
+  if (is_loaded_flat_array) {
+    if (!x->value()->is_null_free()) {
+      __ null_check(value.result(), new CodeEmitInfo(range_check_info));
+    }
+    // If array element is an empty inline type, no need to copy anything
+    if (!x->array()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass()->is_empty()) {
+      access_flat_array(false, array, index, value);
+    }
+  } else {
+    StoreFlattenedArrayStub* slow_path = nullptr;
+
+    if (needs_flat_array_store_check(x)) {
+      // Check if we indeed have a flat array
+      index.load_item();
+      slow_path = new StoreFlattenedArrayStub(array.result(), index.result(), value.result(), state_for(x, x->state_before()));
+      check_flat_array(array.result(), value.result(), slow_path);
+      set_in_conditional_code(true);
+    } else if (needs_null_free_array_store_check(x)) {
+      CodeEmitInfo* info = new CodeEmitInfo(range_check_info);
+      check_null_free_array(array, value, info);
+    }
+
+    DecoratorSet decorators = IN_HEAP | IS_ARRAY;
+    if (x->check_boolean()) {
+      decorators |= C1_MASK_BOOLEAN;
+    }
 
-  access_store_at(decorators, x->elt_type(), array, index.result(), value.result(),
-                  nullptr, null_check_info);
+    access_store_at(decorators, x->elt_type(), array, index.result(), value.result(),
+                    nullptr, null_check_info);
+    if (slow_path != nullptr) {
+      __ branch_destination(slow_path->continuation());
+      set_in_conditional_code(false);
+    }
+  }
 }
 
 void LIRGenerator::access_load_at(DecoratorSet decorators, BasicType type,
                                   LIRItem& base, LIR_Opr offset, LIR_Opr result,
                                   CodeEmitInfo* patch_info, CodeEmitInfo* load_emit_info) {
@@ -1884,10 +2150,44 @@
 
   LIR_Opr result = rlock_result(x, field_type);
   access_load_at(decorators, field_type,
                  object, LIR_OprFact::intConst(x->offset()), result,
                  info ? new CodeEmitInfo(info) : nullptr, info);
+
+  ciField* field = x->field();
+  if (field->is_null_free()) {
+    // Load from non-flat inline type field requires
+    // a null check to replace null with the default value.
+    ciInstanceKlass* holder = field->holder();
+    if (field->is_static() && holder->is_loaded()) {
+      ciObject* val = holder->java_mirror()->field_value(field).as_object();
+      if (!val->is_null_object()) {
+        // Static field is initialized, we don't need to perform a null check.
+        return;
+      }
+    }
+    ciInlineKlass* inline_klass = field->type()->as_inline_klass();
+    if (inline_klass->is_initialized()) {
+      LabelObj* L_end = new LabelObj();
+      __ cmp(lir_cond_notEqual, result, LIR_OprFact::oopConst(nullptr));
+      __ branch(lir_cond_notEqual, L_end->label());
+      set_in_conditional_code(true);
+      Constant* default_value = new Constant(new InstanceConstant(inline_klass->default_instance()));
+      if (default_value->is_pinned()) {
+        __ move(LIR_OprFact::value_type(default_value->type()), result);
+      } else {
+        __ move(load_constant(default_value), result);
+      }
+      __ branch_destination(L_end->label());
+      set_in_conditional_code(false);
+    } else {
+      info = state_for(x, x->state_before());
+      __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(nullptr));
+      __ branch(lir_cond_equal, new DeoptimizeStub(info, Deoptimization::Reason_uninitialized,
+                                                         Deoptimization::Action_make_not_entrant));
+    }
+  }
 }
 
 // int/long jdk.internal.util.Preconditions.checkIndex
 void LIRGenerator::do_PreconditionsCheckIndex(Intrinsic* x, BasicType type) {
   assert(x->number_of_arguments() == 3, "wrong type");
@@ -2028,16 +2328,88 @@
       // The range check performs the null check, so clear it out for the load
       null_check_info = nullptr;
     }
   }
 
-  DecoratorSet decorators = IN_HEAP | IS_ARRAY;
+  ciMethodData* md = nullptr;
+  ciArrayLoadData* load_data = nullptr;
+  if (x->should_profile()) {
+    if (x->array()->is_loaded_flat_array()) {
+      // No need to profile a load from a flat array of known type. This can happen if
+      // the type only became known after optimizations (for example, after the PhiSimplifier).
+      x->set_should_profile(false);
+    } else {
+      int bci = x->profiled_bci();
+      md = x->profiled_method()->method_data();
+      assert(md != nullptr, "Sanity");
+      ciProfileData* data = md->bci_to_data(bci);
+      assert(data != nullptr && data->is_ArrayLoadData(), "incorrect profiling entry");
+      load_data = (ciArrayLoadData*)data;
+      profile_array_type(x, md, load_data);
+    }
+  }
+
+  Value element;
+  if (x->vt() != nullptr) {
+    assert(x->array()->is_loaded_flat_array(), "must be");
+    // Find the destination address (of the NewInlineTypeInstance).
+    LIRItem obj_item(x->vt(), this);
+
+    access_flat_array(true, array, index, obj_item,
+                      x->delayed() == nullptr ? 0 : x->delayed()->field(),
+                      x->delayed() == nullptr ? 0 : x->delayed()->offset());
+    set_no_result(x);
+  } else if (x->delayed() != nullptr) {
+    assert(x->array()->is_loaded_flat_array(), "must be");
+    LIR_Opr result = rlock_result(x, x->delayed()->field()->type()->basic_type());
+    access_sub_element(array, index, result, x->delayed()->field(), x->delayed()->offset());
+  } else if (x->array() != nullptr && x->array()->is_loaded_flat_array() &&
+             x->array()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass()->is_initialized() &&
+             x->array()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass()->is_empty()) {
+    // Load the default instance instead of reading the element
+    ciInlineKlass* elem_klass = x->array()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass();
+    LIR_Opr result = rlock_result(x, x->elt_type());
+    assert(elem_klass->is_initialized(), "Must be");
+    Constant* default_value = new Constant(new InstanceConstant(elem_klass->default_instance()));
+    if (default_value->is_pinned()) {
+      __ move(LIR_OprFact::value_type(default_value->type()), result);
+    } else {
+      __ move(load_constant(default_value), result);
+    }
+  } else {
+    LIR_Opr result = rlock_result(x, x->elt_type());
+    LoadFlattenedArrayStub* slow_path = nullptr;
+
+    if (x->should_profile() && x->array()->maybe_null_free_array()) {
+      profile_null_free_array(array, md, load_data);
+    }
+
+    if (x->elt_type() == T_OBJECT && x->array()->maybe_flat_array()) {
+      assert(x->delayed() == nullptr, "Delayed LoadIndexed only apply to loaded_flat_arrays");
+      index.load_item();
+      // if we are loading from a flat array, load it using a runtime call
+      slow_path = new LoadFlattenedArrayStub(array.result(), index.result(), result, state_for(x, x->state_before()));
+      check_flat_array(array.result(), LIR_OprFact::illegalOpr, slow_path);
+      set_in_conditional_code(true);
+    }
+
+    DecoratorSet decorators = IN_HEAP | IS_ARRAY;
+    access_load_at(decorators, x->elt_type(),
+                   array, index.result(), result,
+                   nullptr, null_check_info);
 
-  LIR_Opr result = rlock_result(x, x->elt_type());
-  access_load_at(decorators, x->elt_type(),
-                 array, index.result(), result,
-                 nullptr, null_check_info);
+    if (slow_path != nullptr) {
+      __ branch_destination(slow_path->continuation());
+      set_in_conditional_code(false);
+    }
+
+    element = x;
+  }
+
+  if (x->should_profile()) {
+    profile_element_type(element, md, load_data);
+  }
 }
 
 
 void LIRGenerator::do_NullCheck(NullCheck* x) {
   if (x->can_trap()) {
@@ -2535,11 +2907,11 @@
   if (!do_null && !do_update) {
     return result;
   }
 
   ciKlass* exact_signature_k = nullptr;
-  if (do_update) {
+  if (do_update && signature_at_call_k != nullptr) {
     // Is the type from the signature exact (the only one possible)?
     exact_signature_k = signature_at_call_k->exact_klass();
     if (exact_signature_k == nullptr) {
       exact_signature_k = comp->cha_exact_type(signature_at_call_k);
     } else {
@@ -2620,10 +2992,50 @@
       }
     }
   }
 }
 
+void LIRGenerator::profile_flags(ciMethodData* md, ciProfileData* data, int flag, LIR_Condition condition) {
+  assert(md != nullptr && data != nullptr, "should have been initialized");
+  LIR_Opr mdp = new_register(T_METADATA);
+  __ metadata2reg(md->constant_encoding(), mdp);
+  LIR_Address* addr = new LIR_Address(mdp, md->byte_offset_of_slot(data, DataLayout::flags_offset()), T_BYTE);
+  LIR_Opr flags = new_register(T_INT);
+  __ move(addr, flags);
+  if (condition != lir_cond_always) {
+    LIR_Opr update = new_register(T_INT);
+    __ cmove(condition, LIR_OprFact::intConst(0), LIR_OprFact::intConst(flag), update, T_INT);
+  } else {
+    __ logical_or(flags, LIR_OprFact::intConst(flag), flags);
+  }
+  __ store(flags, addr);
+}
+
+template <class ArrayData> void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ArrayData* load_store) {
+  assert(compilation()->profile_array_accesses(), "array access profiling is disabled");
+  LabelObj* L_end = new LabelObj();
+  LIR_Opr tmp = new_register(T_METADATA);
+  __ check_null_free_array(array.result(), tmp);
+
+  profile_flags(md, load_store, ArrayStoreData::null_free_array_byte_constant(), lir_cond_equal);
+}
+
+template <class ArrayData> void LIRGenerator::profile_array_type(AccessIndexed* x, ciMethodData*& md, ArrayData*& load_store) {
+  assert(compilation()->profile_array_accesses(), "array access profiling is disabled");
+  LIR_Opr mdp = LIR_OprFact::illegalOpr;
+  profile_type(md, md->byte_offset_of_slot(load_store, ArrayData::array_offset()), 0,
+               load_store->array()->type(), x->array(), mdp, true, nullptr, nullptr);
+}
+
+void LIRGenerator::profile_element_type(Value element, ciMethodData* md, ciArrayLoadData* load_data) {
+  assert(compilation()->profile_array_accesses(), "array access profiling is disabled");
+  assert(md != nullptr && load_data != nullptr, "should have been initialized");
+  LIR_Opr mdp = LIR_OprFact::illegalOpr;
+  profile_type(md, md->byte_offset_of_slot(load_data, ArrayLoadData::element_offset()), 0,
+               load_data->element()->type(), element, mdp, false, nullptr, nullptr);
+}
+
 void LIRGenerator::do_Base(Base* x) {
   __ std_entry(LIR_OprFact::illegalOpr);
   // Emit moves from physical registers / stack slots to virtual registers
   CallingConvention* args = compilation()->frame_map()->incoming_arguments();
   IRScope* irScope = compilation()->hir()->top_scope();
@@ -2702,10 +3114,18 @@
   if (!method()->is_accessor()) { // Accessors do not have MDOs, so no counting.
     profile_parameters(x);
     CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, SynchronizationEntryBCI), nullptr, false);
     increment_invocation_counter(info);
   }
+  if (method()->has_scalarized_args()) {
+    // Check if deoptimization was triggered (i.e. orig_pc was set) while buffering scalarized inline type arguments
+    // in the entry point (see comments in frame::deoptimize). If so, deoptimize only now that we have the right state.
+    CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, 0), nullptr, false);
+    CodeStub* deopt_stub = new DeoptimizeStub(info, Deoptimization::Reason_none, Deoptimization::Action_none);
+    __ append(new LIR_Op0(lir_check_orig_pc));
+    __ branch(lir_cond_notEqual, deopt_stub);
+  }
 
   // all blocks with a successor must end with an unconditional jump
   // to the successor even if they are consecutive
   __ jump(x->default_sux());
 }
@@ -2717,27 +3137,31 @@
   __ osr_entry(LIR_Assembler::osrBufferPointer());
   LIR_Opr result = rlock_result(x);
   __ move(LIR_Assembler::osrBufferPointer(), result);
 }
 
+void LIRGenerator::invoke_load_one_argument(LIRItem* param, LIR_Opr loc) {
+  if (loc->is_register()) {
+    param->load_item_force(loc);
+  } else {
+    LIR_Address* addr = loc->as_address_ptr();
+    param->load_for_store(addr->type());
+    if (addr->type() == T_OBJECT) {
+      __ move_wide(param->result(), addr);
+    } else {
+      __ move(param->result(), addr);
+    }
+  }
+}
 
 void LIRGenerator::invoke_load_arguments(Invoke* x, LIRItemList* args, const LIR_OprList* arg_list) {
   assert(args->length() == arg_list->length(),
          "args=%d, arg_list=%d", args->length(), arg_list->length());
   for (int i = x->has_receiver() ? 1 : 0; i < args->length(); i++) {
     LIRItem* param = args->at(i);
     LIR_Opr loc = arg_list->at(i);
-    if (loc->is_register()) {
-      param->load_item_force(loc);
-    } else {
-      LIR_Address* addr = loc->as_address_ptr();
-      param->load_for_store(addr->type());
-      if (addr->type() == T_OBJECT) {
-        __ move_wide(param->result(), addr);
-      } else
-        __ move(param->result(), addr);
-    }
+    invoke_load_one_argument(param, loc);
   }
 
   if (x->has_receiver()) {
     LIRItem* receiver = args->at(0);
     LIR_Opr loc = arg_list->at(0);
@@ -2899,24 +3323,82 @@
 #endif
 
   LIRItem left(x->x(), this);
   LIRItem right(x->y(), this);
   left.load_item();
-  if (can_inline_as_constant(right.value())) {
+  if (can_inline_as_constant(right.value()) && !x->substitutability_check()) {
     right.dont_load_item();
   } else {
+    // substitutability_check() needs to use right as a base register.
     right.load_item();
   }
 
   LIRItem t_val(x->tval(), this);
   LIRItem f_val(x->fval(), this);
   t_val.dont_load_item();
   f_val.dont_load_item();
-  LIR_Opr reg = rlock_result(x);
 
-  __ cmp(lir_cond(x->cond()), left.result(), right.result());
-  __ cmove(lir_cond(x->cond()), t_val.result(), f_val.result(), reg, as_BasicType(x->x()->type()));
+  if (x->substitutability_check()) {
+    substitutability_check(x, left, right, t_val, f_val);
+  } else {
+    LIR_Opr reg = rlock_result(x);
+    __ cmp(lir_cond(x->cond()), left.result(), right.result());
+    __ cmove(lir_cond(x->cond()), t_val.result(), f_val.result(), reg, as_BasicType(x->x()->type()));
+  }
+}
+
+void LIRGenerator::substitutability_check(IfOp* x, LIRItem& left, LIRItem& right, LIRItem& t_val, LIRItem& f_val) {
+  assert(x->cond() == If::eql || x->cond() == If::neq, "must be");
+  bool is_acmpeq = (x->cond() == If::eql);
+  LIR_Opr equal_result     = is_acmpeq ? t_val.result() : f_val.result();
+  LIR_Opr not_equal_result = is_acmpeq ? f_val.result() : t_val.result();
+  LIR_Opr result = rlock_result(x);
+  CodeEmitInfo* info = state_for(x, x->state_before());
+
+  substitutability_check_common(x->x(), x->y(), left, right, equal_result, not_equal_result, result, info);
+}
+
+void LIRGenerator::substitutability_check(If* x, LIRItem& left, LIRItem& right) {
+  LIR_Opr equal_result     = LIR_OprFact::intConst(1);
+  LIR_Opr not_equal_result = LIR_OprFact::intConst(0);
+  LIR_Opr result = new_register(T_INT);
+  CodeEmitInfo* info = state_for(x, x->state_before());
+
+  substitutability_check_common(x->x(), x->y(), left, right, equal_result, not_equal_result, result, info);
+
+  assert(x->cond() == If::eql || x->cond() == If::neq, "must be");
+  __ cmp(lir_cond(x->cond()), result, equal_result);
+}
+
+void LIRGenerator::substitutability_check_common(Value left_val, Value right_val, LIRItem& left, LIRItem& right,
+                                                 LIR_Opr equal_result, LIR_Opr not_equal_result, LIR_Opr result,
+                                                 CodeEmitInfo* info) {
+  LIR_Opr tmp1 = LIR_OprFact::illegalOpr;
+  LIR_Opr tmp2 = LIR_OprFact::illegalOpr;
+  LIR_Opr left_klass_op = LIR_OprFact::illegalOpr;
+  LIR_Opr right_klass_op = LIR_OprFact::illegalOpr;
+
+  ciKlass* left_klass  = left_val ->as_loaded_klass_or_null();
+  ciKlass* right_klass = right_val->as_loaded_klass_or_null();
+
+  if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
+      !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
+    init_temps_for_substitutability_check(tmp1, tmp2);
+  }
+
+  if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
+    // No need to load klass -- the operands are statically known to be the same inline klass.
+  } else {
+    BasicType t_klass = UseCompressedOops ? T_INT : T_METADATA;
+    left_klass_op = new_register(t_klass);
+    right_klass_op = new_register(t_klass);
+  }
+
+  CodeStub* slow_path = new SubstitutabilityCheckStub(left.result(), right.result(), info);
+  __ substitutability_check(result, left.result(), right.result(), equal_result, not_equal_result,
+                            tmp1, tmp2,
+                            left_klass, right_klass, left_klass_op, right_klass_op, info, slow_path);
 }
 
 void LIRGenerator::do_RuntimeCall(address routine, Intrinsic* x) {
   assert(x->number_of_arguments() == 0, "wrong type");
   // Enforce computation of _reserved_argument_area_size which is required on some platforms.
@@ -3186,11 +3668,11 @@
   ciMethodData* md = x->method()->method_data_or_null();
   assert(md != nullptr, "Sanity");
   ciProfileData* data = md->bci_to_data(bci);
   if (data != nullptr) {
     assert(data->is_CallTypeData() || data->is_VirtualCallTypeData(), "wrong profile data type");
-    ciReturnTypeEntry* ret = data->is_CallTypeData() ? ((ciCallTypeData*)data)->ret() : ((ciVirtualCallTypeData*)data)->ret();
+    ciSingleTypeEntry* ret = data->is_CallTypeData() ? ((ciCallTypeData*)data)->ret() : ((ciVirtualCallTypeData*)data)->ret();
     LIR_Opr mdp = LIR_OprFact::illegalOpr;
 
     bool ignored_will_link;
     ciSignature* signature_at_call = nullptr;
     x->method()->get_method_at_bci(bci, ignored_will_link, &signature_at_call);
@@ -3207,10 +3689,57 @@
       md->set_return_type(bci, exact);
     }
   }
 }
 
+bool LIRGenerator::profile_inline_klass(ciMethodData* md, ciProfileData* data, Value value, int flag) {
+  ciKlass* klass = value->as_loaded_klass_or_null();
+  if (klass != nullptr) {
+    if (klass->is_inlinetype()) {
+      profile_flags(md, data, flag, lir_cond_always);
+    } else if (klass->can_be_inline_klass()) {
+      return false;
+    }
+  } else {
+    return false;
+  }
+  return true;
+}
+
+
+void LIRGenerator::do_ProfileACmpTypes(ProfileACmpTypes* x) {
+  ciMethod* method = x->method();
+  assert(method != nullptr, "method should be set if branch is profiled");
+  ciMethodData* md = method->method_data_or_null();
+  assert(md != nullptr, "Sanity");
+  ciProfileData* data = md->bci_to_data(x->bci());
+  assert(data != nullptr, "must have profiling data");
+  assert(data->is_ACmpData(), "need BranchData for two-way branches");
+  ciACmpData* acmp = (ciACmpData*)data;
+  LIR_Opr mdp = LIR_OprFact::illegalOpr;
+  profile_type(md, md->byte_offset_of_slot(acmp, ACmpData::left_offset()), 0,
+               acmp->left()->type(), x->left(), mdp, !x->left_maybe_null(), nullptr, nullptr);
+  int flags_offset = md->byte_offset_of_slot(data, DataLayout::flags_offset());
+  if (!profile_inline_klass(md, acmp, x->left(), ACmpData::left_inline_type_byte_constant())) {
+    LIR_Opr mdp = new_register(T_METADATA);
+    __ metadata2reg(md->constant_encoding(), mdp);
+    LIRItem value(x->left(), this);
+    value.load_item();
+    __ profile_inline_type(new LIR_Address(mdp, flags_offset, T_INT), value.result(), ACmpData::left_inline_type_byte_constant(), new_register(T_INT), !x->left_maybe_null());
+  }
+  profile_type(md, md->byte_offset_of_slot(acmp, ACmpData::left_offset()),
+               in_bytes(ACmpData::right_offset()) - in_bytes(ACmpData::left_offset()),
+               acmp->right()->type(), x->right(), mdp, !x->right_maybe_null(), nullptr, nullptr);
+  if (!profile_inline_klass(md, acmp, x->right(), ACmpData::right_inline_type_byte_constant())) {
+    LIR_Opr mdp = new_register(T_METADATA);
+    __ metadata2reg(md->constant_encoding(), mdp);
+    LIRItem value(x->right(), this);
+    value.load_item();
+    __ profile_inline_type(new LIR_Address(mdp, flags_offset, T_INT), value.result(), ACmpData::right_inline_type_byte_constant(), new_register(T_INT), !x->left_maybe_null());
+  }
+}
+
 void LIRGenerator::do_ProfileInvoke(ProfileInvoke* x) {
   // We can safely ignore accessors here, since c2 will inline them anyway,
   // accessors are also always mature.
   if (!x->inlinee()->is_accessor()) {
     CodeEmitInfo* info = state_for(x, x->state(), true);
