< prev index next >

src/hotspot/share/c1/c1_LIRGenerator.cpp

Print this page
@@ -29,10 +29,12 @@
  #include "c1/c1_Instruction.hpp"
  #include "c1/c1_LIRAssembler.hpp"
  #include "c1/c1_LIRGenerator.hpp"
  #include "c1/c1_ValueStack.hpp"
  #include "ci/ciArrayKlass.hpp"
+ #include "ci/ciFlatArrayKlass.hpp"
+ #include "ci/ciInlineKlass.hpp"
  #include "ci/ciInstance.hpp"
  #include "ci/ciObjArray.hpp"
  #include "ci/ciUtilities.hpp"
  #include "gc/shared/barrierSet.hpp"
  #include "gc/shared/c1/barrierSetC1.hpp"

@@ -208,10 +210,12 @@
  
    _result = opr;
  }
  
  void LIRItem::load_item() {
+   assert(!_gen->in_conditional_code(), "LIRItem cannot be loaded in conditional code");
+ 
    if (result()->is_illegal()) {
      // update the items result
      _result = value()->operand();
    }
    if (!result()->is_register()) {

@@ -603,17 +607,18 @@
      default: ShouldNotReachHere();
    }
  }
  
  
- void LIRGenerator::monitor_enter(LIR_Opr object, LIR_Opr lock, LIR_Opr hdr, LIR_Opr scratch, int monitor_no, CodeEmitInfo* info_for_exception, CodeEmitInfo* info) {
+ void LIRGenerator::monitor_enter(LIR_Opr object, LIR_Opr lock, LIR_Opr hdr, LIR_Opr scratch, int monitor_no,
+                                  CodeEmitInfo* info_for_exception, CodeEmitInfo* info, CodeStub* throw_imse_stub) {
    if (!GenerateSynchronizationCode) return;
    // for slow path, use debug info for state after successful locking
-   CodeStub* slow_path = new MonitorEnterStub(object, lock, info);
+   CodeStub* slow_path = new MonitorEnterStub(object, lock, info, throw_imse_stub, scratch);
    __ load_stack_address_monitor(monitor_no, lock);
    // for handling NullPointerException, use debug info representing just the lock stack before this monitorenter
-   __ lock_object(hdr, object, lock, scratch, slow_path, info_for_exception);
+   __ lock_object(hdr, object, lock, scratch, slow_path, info_for_exception, throw_imse_stub);
  }
  
  
  void LIRGenerator::monitor_exit(LIR_Opr object, LIR_Opr lock, LIR_Opr new_hdr, LIR_Opr scratch, int monitor_no) {
    if (!GenerateSynchronizationCode) return;

@@ -633,14 +638,19 @@
      tty->print_cr("   ###class not resolved at new bci %d", new_instance->printable_bci());
    }
  }
  #endif
  
- void LIRGenerator::new_instance(LIR_Opr dst, ciInstanceKlass* klass, bool is_unresolved, LIR_Opr scratch1, LIR_Opr scratch2, LIR_Opr scratch3, LIR_Opr scratch4, LIR_Opr klass_reg, CodeEmitInfo* info) {
-   klass2reg_with_patching(klass_reg, klass, info, is_unresolved);
-   // If klass is not loaded we do not know if the klass has finalizers:
-   if (UseFastNewInstance && klass->is_loaded()
+ void LIRGenerator::new_instance(LIR_Opr dst, ciInstanceKlass* klass, bool is_unresolved, bool allow_inline, LIR_Opr scratch1, LIR_Opr scratch2, LIR_Opr scratch3, LIR_Opr scratch4, LIR_Opr klass_reg, CodeEmitInfo* info) {
+   if (allow_inline) {
+     assert(!is_unresolved && klass->is_loaded(), "inline type klass should be resolved");
+     __ metadata2reg(klass->constant_encoding(), klass_reg);
+   } else {
+     klass2reg_with_patching(klass_reg, klass, info, is_unresolved);
+   }
+   // If klass is not loaded we do not know if the klass has finalizers or is an unexpected inline klass
+   if (UseFastNewInstance && klass->is_loaded() && (allow_inline || !klass->is_inlinetype())
        && !Klass::layout_helper_needs_slow_path(klass->layout_helper())) {
  
      Runtime1::StubID stub_id = klass->is_initialized() ? Runtime1::fast_new_instance_id : Runtime1::fast_new_instance_init_check_id;
  
      CodeStub* slow_path = new NewInstanceStub(klass_reg, dst, klass, info, stub_id);

@@ -650,12 +660,12 @@
      assert(klass->size_helper() > 0, "illegal instance size");
      const int instance_size = align_object_size(klass->size_helper());
      __ allocate_object(dst, scratch1, scratch2, scratch3, scratch4,
                         oopDesc::header_size(), instance_size, klass_reg, !klass->is_initialized(), slow_path);
    } else {
-     CodeStub* slow_path = new NewInstanceStub(klass_reg, dst, klass, info, Runtime1::new_instance_id);
-     __ branch(lir_cond_always, slow_path);
+     CodeStub* slow_path = new NewInstanceStub(klass_reg, dst, klass, info, allow_inline ? Runtime1::new_instance_id : Runtime1::new_instance_no_inline_id);
+     __ jump(slow_path);
      __ branch_destination(slow_path->continuation());
    }
  }
  
  

@@ -751,10 +761,20 @@
  
    // if a probable array type has been identified, figure out if any
    // of the required checks for a fast case can be elided.
    int flags = LIR_OpArrayCopy::all_flags;
  
+   if (!src->is_loaded_flattened_array() && !dst->is_loaded_flattened_array()) {
+     flags &= ~LIR_OpArrayCopy::always_slow_path;
+   }
+   if (!src->maybe_flattened_array()) {
+     flags &= ~LIR_OpArrayCopy::src_inlinetype_check;
+   }
+   if (!dst->maybe_flattened_array() && !dst->maybe_null_free_array()) {
+     flags &= ~LIR_OpArrayCopy::dst_inlinetype_check;
+   }
+ 
    if (!src_objarray)
      flags &= ~LIR_OpArrayCopy::src_objarray;
    if (!dst_objarray)
      flags &= ~LIR_OpArrayCopy::dst_objarray;
  

@@ -1523,15 +1543,23 @@
      }
    }
  
    LIR_Opr result = new_register(t);
    __ move((LIR_Opr)c, result);
-   _constants.append(c);
-   _reg_for_constants.append(result);
+   if (!in_conditional_code()) {
+     _constants.append(c);
+     _reg_for_constants.append(result);
+   }
    return result;
  }
  
+ void LIRGenerator::set_in_conditional_code(bool v) {
+   assert(v != _in_conditional_code, "must change state");
+   _in_conditional_code = v;
+ }
+ 
+ 
  //------------------------field access--------------------------------------
  
  void LIRGenerator::do_CompareAndSwap(Intrinsic* x, ValueType* type) {
    assert(x->number_of_arguments() == 4, "wrong type");
    LIRItem obj   (x->argument_at(0), this);  // object

@@ -1619,10 +1647,15 @@
      tty->print_cr("   ###class not loaded at store_%s bci %d",
                    x->is_static() ?  "static" : "field", x->printable_bci());
    }
  #endif
  
+   if (!inline_type_field_access_prolog(x)) {
+     // Field store will always deopt due to unloaded field or holder klass
+     return;
+   }
+ 
    if (x->needs_null_check() &&
        (needs_patching ||
         MacroAssembler::needs_explicit_null_check(x->offset()))) {
      // Emit an explicit null check because the offset is too large.
      // If the class is not loaded and the object is NULL, we need to deoptimize to throw a

@@ -1640,18 +1673,191 @@
  
    access_store_at(decorators, field_type, object, LIR_OprFact::intConst(x->offset()),
                    value.result(), info != NULL ? new CodeEmitInfo(info) : NULL, info);
  }
  
+ // FIXME -- I can't find any other way to pass an address to access_load_at().
+ class TempResolvedAddress: public Instruction {
+  public:
+   TempResolvedAddress(ValueType* type, LIR_Opr addr) : Instruction(type) {
+     set_operand(addr);
+   }
+   virtual void input_values_do(ValueVisitor*) {}
+   virtual void visit(InstructionVisitor* v)   {}
+   virtual const char* name() const  { return "TempResolvedAddress"; }
+ };
+ 
+ LIR_Opr LIRGenerator::get_and_load_element_address(LIRItem& array, LIRItem& index) {
+   ciType* array_type = array.value()->declared_type();
+   ciFlatArrayKlass* flat_array_klass = array_type->as_flat_array_klass();
+   assert(flat_array_klass->is_loaded(), "must be");
+ 
+   int array_header_size = flat_array_klass->array_header_in_bytes();
+   int shift = flat_array_klass->log2_element_size();
+ 
+ #ifndef _LP64
+   LIR_Opr index_op = new_register(T_INT);
+   // FIXME -- on 32-bit, the shift below can overflow, so we need to check that
+   // the top (shift+1) bits of index_op must be zero, or
+   // else throw ArrayIndexOutOfBoundsException
+   if (index.result()->is_constant()) {
+     jint const_index = index.result()->as_jint();
+     __ move(LIR_OprFact::intConst(const_index << shift), index_op);
+   } else {
+     __ shift_left(index_op, shift, index.result());
+   }
+ #else
+   LIR_Opr index_op = new_register(T_LONG);
+   if (index.result()->is_constant()) {
+     jint const_index = index.result()->as_jint();
+     __ move(LIR_OprFact::longConst(const_index << shift), index_op);
+   } else {
+     __ convert(Bytecodes::_i2l, index.result(), index_op);
+     // Need to shift manually, as LIR_Address can scale only up to 3.
+     __ shift_left(index_op, shift, index_op);
+   }
+ #endif
+ 
+   LIR_Opr elm_op = new_pointer_register();
+   LIR_Address* elm_address = generate_address(array.result(), index_op, 0, array_header_size, T_ADDRESS);
+   __ leal(LIR_OprFact::address(elm_address), elm_op);
+   return elm_op;
+ }
+ 
+ void LIRGenerator::access_sub_element(LIRItem& array, LIRItem& index, LIR_Opr& result, ciField* field, int sub_offset) {
+   assert(field != NULL, "Need a subelement type specified");
+ 
+   // Find the starting address of the source (inside the array)
+   LIR_Opr elm_op = get_and_load_element_address(array, index);
+ 
+   BasicType subelt_type = field->type()->basic_type();
+   TempResolvedAddress* elm_resolved_addr = new TempResolvedAddress(as_ValueType(subelt_type), elm_op);
+   LIRItem elm_item(elm_resolved_addr, this);
+ 
+   DecoratorSet decorators = IN_HEAP;
+   access_load_at(decorators, subelt_type,
+                      elm_item, LIR_OprFact::intConst(sub_offset), result,
+                      NULL, NULL);
+ 
+   if (field->is_null_free()) {
+     assert(field->type()->as_inline_klass()->is_loaded(), "Must be");
+     LabelObj* L_end = new LabelObj();
+     __ cmp(lir_cond_notEqual, result, LIR_OprFact::oopConst(NULL));
+     __ branch(lir_cond_notEqual, L_end->label());
+     set_in_conditional_code(true);
+     Constant* default_value = new Constant(new InstanceConstant(field->type()->as_inline_klass()->default_instance()));
+     if (default_value->is_pinned()) {
+       __ move(LIR_OprFact::value_type(default_value->type()), result);
+     } else {
+       __ move(load_constant(default_value), result);
+     }
+     __ branch_destination(L_end->label());
+     set_in_conditional_code(false);
+   }
+ }
+ 
+ void LIRGenerator::access_flattened_array(bool is_load, LIRItem& array, LIRItem& index, LIRItem& obj_item,
+                                           ciField* field, int sub_offset) {
+   assert(sub_offset == 0 || field != NULL, "Sanity check");
+ 
+   // Find the starting address of the source (inside the array)
+   LIR_Opr elm_op = get_and_load_element_address(array, index);
+ 
+   ciInlineKlass* elem_klass = NULL;
+   if (field != NULL) {
+     elem_klass = field->type()->as_inline_klass();
+   } else {
+     elem_klass = array.value()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass();
+   }
+   for (int i = 0; i < elem_klass->nof_nonstatic_fields(); i++) {
+     ciField* inner_field = elem_klass->nonstatic_field_at(i);
+     assert(!inner_field->is_flattened(), "flattened fields must have been expanded");
+     int obj_offset = inner_field->offset();
+     int elm_offset = obj_offset - elem_klass->first_field_offset() + sub_offset; // object header is not stored in array.
+     BasicType field_type = inner_field->type()->basic_type();
+ 
+     // Types which are smaller than int are still passed in an int register.
+     BasicType reg_type = field_type;
+     switch (reg_type) {
+     case T_BYTE:
+     case T_BOOLEAN:
+     case T_SHORT:
+     case T_CHAR:
+       reg_type = T_INT;
+       break;
+     default:
+       break;
+     }
+ 
+     LIR_Opr temp = new_register(reg_type);
+     TempResolvedAddress* elm_resolved_addr = new TempResolvedAddress(as_ValueType(field_type), elm_op);
+     LIRItem elm_item(elm_resolved_addr, this);
+ 
+     DecoratorSet decorators = IN_HEAP;
+     if (is_load) {
+       access_load_at(decorators, field_type,
+                      elm_item, LIR_OprFact::intConst(elm_offset), temp,
+                      NULL, NULL);
+       access_store_at(decorators, field_type,
+                       obj_item, LIR_OprFact::intConst(obj_offset), temp,
+                       NULL, NULL);
+     } else {
+       access_load_at(decorators, field_type,
+                      obj_item, LIR_OprFact::intConst(obj_offset), temp,
+                      NULL, NULL);
+       access_store_at(decorators, field_type,
+                       elm_item, LIR_OprFact::intConst(elm_offset), temp,
+                       NULL, NULL);
+     }
+   }
+ }
+ 
+ void LIRGenerator::check_flattened_array(LIR_Opr array, LIR_Opr value, CodeStub* slow_path) {
+   LIR_Opr tmp = new_register(T_METADATA);
+   __ check_flattened_array(array, value, tmp, slow_path);
+ }
+ 
+ void LIRGenerator::check_null_free_array(LIRItem& array, LIRItem& value, CodeEmitInfo* info) {
+   LabelObj* L_end = new LabelObj();
+   LIR_Opr tmp = new_register(T_METADATA);
+   __ check_null_free_array(array.result(), tmp);
+   __ branch(lir_cond_equal, L_end->label());
+   __ null_check(value.result(), info);
+   __ branch_destination(L_end->label());
+ }
+ 
+ bool LIRGenerator::needs_flattened_array_store_check(StoreIndexed* x) {
+   if (x->elt_type() == T_OBJECT && x->array()->maybe_flattened_array()) {
+     ciType* type = x->value()->declared_type();
+     if (type != NULL && type->is_klass()) {
+       ciKlass* klass = type->as_klass();
+       if (!klass->can_be_inline_klass() || (klass->is_inlinetype() && !klass->as_inline_klass()->flatten_array())) {
+         // This is known to be a non-flattened object. If the array is flattened,
+         // it will be caught by the code generated by array_store_check().
+         return false;
+       }
+     }
+     // We're not 100% sure, so let's do the flattened_array_store_check.
+     return true;
+   }
+   return false;
+ }
+ 
+ bool LIRGenerator::needs_null_free_array_store_check(StoreIndexed* x) {
+   return x->elt_type() == T_OBJECT && x->array()->maybe_null_free_array();
+ }
+ 
  void LIRGenerator::do_StoreIndexed(StoreIndexed* x) {
    assert(x->is_pinned(),"");
+   assert(x->elt_type() != T_ARRAY, "never used");
+   bool is_loaded_flattened_array = x->array()->is_loaded_flattened_array();
    bool needs_range_check = x->compute_needs_range_check();
    bool use_length = x->length() != NULL;
    bool obj_store = is_reference_type(x->elt_type());
-   bool needs_store_check = obj_store && (x->value()->as_Constant() == NULL ||
-                                          !get_jobject_constant(x->value())->is_null_object() ||
-                                          x->should_profile());
+   bool needs_store_check = obj_store && !(is_loaded_flattened_array && x->is_exact_flattened_array_store()) &&
+                                         (x->value()->as_Constant() == NULL ||
+                                          !get_jobject_constant(x->value())->is_null_object());
  
    LIRItem array(x->array(), this);
    LIRItem index(x->index(), this);
    LIRItem value(x->value(), this);
    LIRItem length(this);

@@ -1660,13 +1866,14 @@
    index.load_nonconstant();
  
    if (use_length && needs_range_check) {
      length.set_instruction(x->length());
      length.load_item();
- 
    }
-   if (needs_store_check || x->check_boolean()) {
+ 
+   if (needs_store_check || x->check_boolean()
+       || is_loaded_flattened_array || needs_flattened_array_store_check(x) || needs_null_free_array_store_check(x)) {
      value.load_item();
    } else {
      value.load_for_store(x->elt_type());
    }
  

@@ -1690,22 +1897,65 @@
        // range_check also does the null check
        null_check_info = NULL;
      }
    }
  
+   if (x->should_profile()) {
+     if (x->array()->is_loaded_flattened_array()) {
+       // No need to profile a store to a flattened array of known type. This can happen if
+       // the type only became known after optimizations (for example, after the PhiSimplifier).
+       x->set_should_profile(false);
+     } else {
+       ciMethodData* md = NULL;
+       ciArrayLoadStoreData* load_store = NULL;
+       profile_array_type(x, md, load_store);
+       if (x->array()->maybe_null_free_array()) {
+         profile_null_free_array(array, md, load_store);
+       }
+       profile_element_type(x->value(), md, load_store);
+     }
+   }
+ 
    if (GenerateArrayStoreCheck && needs_store_check) {
      CodeEmitInfo* store_check_info = new CodeEmitInfo(range_check_info);
-     array_store_check(value.result(), array.result(), store_check_info, x->profiled_method(), x->profiled_bci());
+     array_store_check(value.result(), array.result(), store_check_info, NULL, -1);
    }
  
-   DecoratorSet decorators = IN_HEAP | IS_ARRAY;
-   if (x->check_boolean()) {
-     decorators |= C1_MASK_BOOLEAN;
-   }
+   if (is_loaded_flattened_array) {
+     if (!x->value()->is_null_free()) {
+       __ null_check(value.result(), new CodeEmitInfo(range_check_info));
+     }
+     // If array element is an empty inline type, no need to copy anything
+     if (!x->array()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass()->is_empty()) {
+       access_flattened_array(false, array, index, value);
+     }
+   } else {
+     StoreFlattenedArrayStub* slow_path = NULL;
  
-   access_store_at(decorators, x->elt_type(), array, index.result(), value.result(),
-                   NULL, null_check_info);
+     if (needs_flattened_array_store_check(x)) {
+       // Check if we indeed have a flattened array
+       index.load_item();
+       slow_path = new StoreFlattenedArrayStub(array.result(), index.result(), value.result(), state_for(x, x->state_before()));
+       check_flattened_array(array.result(), value.result(), slow_path);
+       set_in_conditional_code(true);
+     } else if (needs_null_free_array_store_check(x)) {
+       CodeEmitInfo* info = new CodeEmitInfo(range_check_info);
+       check_null_free_array(array, value, info);
+     }
+ 
+     DecoratorSet decorators = IN_HEAP | IS_ARRAY;
+     if (x->check_boolean()) {
+       decorators |= C1_MASK_BOOLEAN;
+     }
+ 
+     access_store_at(decorators, x->elt_type(), array, index.result(), value.result(),
+                     NULL, null_check_info);
+     if (slow_path != NULL) {
+       __ branch_destination(slow_path->continuation());
+       set_in_conditional_code(false);
+     }
+   }
  }
  
  void LIRGenerator::access_load_at(DecoratorSet decorators, BasicType type,
                                    LIRItem& base, LIR_Opr offset, LIR_Opr result,
                                    CodeEmitInfo* patch_info, CodeEmitInfo* load_emit_info) {

@@ -1782,10 +2032,35 @@
    } else {
      return _barrier_set->atomic_add_at(access, value);
    }
  }
  
+ bool LIRGenerator::inline_type_field_access_prolog(AccessField* x) {
+   ciField* field = x->field();
+   assert(!field->is_flattened(), "Flattened field access should have been expanded");
+   if (!field->is_null_free()) {
+     return true; // Not an inline type field
+   }
+   // Deoptimize if the access is non-static and requires patching (holder not loaded
+   // or not accessible) because then we only have partial field information and the
+   // field could be flattened (see ciField constructor).
+   bool could_be_flat = !x->is_static() && x->needs_patching();
+   // Deoptimize if we load from a static field with an uninitialized type because we
+   // need to throw an exception if initialization of the type failed.
+   bool not_initialized = x->is_static() && x->as_LoadField() != NULL &&
+       !field->type()->as_instance_klass()->is_initialized();
+   if (could_be_flat || not_initialized) {
+     CodeEmitInfo* info = state_for(x, x->state_before());
+     CodeStub* stub = new DeoptimizeStub(new CodeEmitInfo(info),
+                                         Deoptimization::Reason_unloaded,
+                                         Deoptimization::Action_make_not_entrant);
+     __ jump(stub);
+     return false;
+   }
+   return true;
+ }
+ 
  void LIRGenerator::do_LoadField(LoadField* x) {
    bool needs_patching = x->needs_patching();
    bool is_volatile = x->field()->is_volatile();
    BasicType field_type = x->field_type();
  

@@ -1811,10 +2086,17 @@
      tty->print_cr("   ###class not loaded at load_%s bci %d",
                    x->is_static() ?  "static" : "field", x->printable_bci());
    }
  #endif
  
+   if (!inline_type_field_access_prolog(x)) {
+     // Field load will always deopt due to unloaded field or holder klass
+     LIR_Opr result = rlock_result(x, field_type);
+     __ move(LIR_OprFact::oopConst(NULL), result);
+     return;
+   }
+ 
    bool stress_deopt = StressLoopInvariantCodeMotion && info && info->deoptimize_on_exception();
    if (x->needs_null_check() &&
        (needs_patching ||
         MacroAssembler::needs_explicit_null_check(x->offset()) ||
         stress_deopt)) {

@@ -1839,10 +2121,37 @@
  
    LIR_Opr result = rlock_result(x, field_type);
    access_load_at(decorators, field_type,
                   object, LIR_OprFact::intConst(x->offset()), result,
                   info ? new CodeEmitInfo(info) : NULL, info);
+ 
+   ciField* field = x->field();
+   if (field->is_null_free()) {
+     // Load from non-flattened inline type field requires
+     // a null check to replace null with the default value.
+     ciInstanceKlass* holder = field->holder();
+     if (field->is_static() && holder->is_loaded()) {
+       ciObject* val = holder->java_mirror()->field_value(field).as_object();
+       if (!val->is_null_object()) {
+         // Static field is initialized, we don need to perform a null check.
+         return;
+       }
+     }
+     LabelObj* L_end = new LabelObj();
+     __ cmp(lir_cond_notEqual, result, LIR_OprFact::oopConst(NULL));
+     __ branch(lir_cond_notEqual, L_end->label());
+     set_in_conditional_code(true);
+     ciInlineKlass* inline_klass = field->type()->as_inline_klass();
+     Constant* default_value = new Constant(new InstanceConstant(inline_klass->default_instance()));
+     if (default_value->is_pinned()) {
+       __ move(LIR_OprFact::value_type(default_value->type()), result);
+     } else {
+       __ move(load_constant(default_value), result);
+     }
+     __ branch_destination(L_end->label());
+     set_in_conditional_code(false);
+   }
  }
  
  // int/long jdk.internal.util.Preconditions.checkIndex
  void LIRGenerator::do_PreconditionsCheckIndex(Intrinsic* x, BasicType type) {
    assert(x->number_of_arguments() == 3, "wrong type");

@@ -1983,18 +2292,94 @@
        // The range check performs the null check, so clear it out for the load
        null_check_info = NULL;
      }
    }
  
-   DecoratorSet decorators = IN_HEAP | IS_ARRAY;
+   ciMethodData* md = NULL;
+   ciArrayLoadStoreData* load_store = NULL;
+   if (x->should_profile()) {
+     if (x->array()->is_loaded_flattened_array()) {
+       // No need to profile a load from a flattened array of known type. This can happen if
+       // the type only became known after optimizations (for example, after the PhiSimplifier).
+       x->set_should_profile(false);
+     } else {
+       profile_array_type(x, md, load_store);
+     }
+   }
+ 
+   Value element;
+   if (x->vt() != NULL) {
+     assert(x->array()->is_loaded_flattened_array(), "must be");
+     // Find the destination address (of the NewInlineTypeInstance).
+     LIRItem obj_item(x->vt(), this);
+ 
+     access_flattened_array(true, array, index, obj_item,
+                            x->delayed() == NULL ? 0 : x->delayed()->field(),
+                            x->delayed() == NULL ? 0 : x->delayed()->offset());
+     set_no_result(x);
+   } else if (x->delayed() != NULL) {
+     assert(x->array()->is_loaded_flattened_array(), "must be");
+     LIR_Opr result = rlock_result(x, x->delayed()->field()->type()->basic_type());
+     access_sub_element(array, index, result, x->delayed()->field(), x->delayed()->offset());
+   } else if (x->array() != NULL && x->array()->is_loaded_flattened_array() &&
+              x->array()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass()->is_empty()) {
+     // Load the default instance instead of reading the element
+     ciInlineKlass* elem_klass = x->array()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass();
+     LIR_Opr result = rlock_result(x, x->elt_type());
+     Constant* default_value = new Constant(new InstanceConstant(elem_klass->default_instance()));
+     if (default_value->is_pinned()) {
+       __ move(LIR_OprFact::value_type(default_value->type()), result);
+     } else {
+       __ move(load_constant(default_value), result);
+     }
+   } else {
+     LIR_Opr result = rlock_result(x, x->elt_type());
+     LoadFlattenedArrayStub* slow_path = NULL;
  
-   LIR_Opr result = rlock_result(x, x->elt_type());
-   access_load_at(decorators, x->elt_type(),
-                  array, index.result(), result,
-                  NULL, null_check_info);
+     if (x->should_profile() && x->array()->maybe_null_free_array()) {
+       profile_null_free_array(array, md, load_store);
+     }
+ 
+     if (x->elt_type() == T_OBJECT && x->array()->maybe_flattened_array()) {
+       assert(x->delayed() == NULL, "Delayed LoadIndexed only apply to loaded_flattened_arrays");
+       index.load_item();
+       // if we are loading from flattened array, load it using a runtime call
+       slow_path = new LoadFlattenedArrayStub(array.result(), index.result(), result, state_for(x, x->state_before()));
+       check_flattened_array(array.result(), LIR_OprFact::illegalOpr, slow_path);
+       set_in_conditional_code(true);
+     }
+ 
+     DecoratorSet decorators = IN_HEAP | IS_ARRAY;
+     access_load_at(decorators, x->elt_type(),
+                    array, index.result(), result,
+                    NULL, null_check_info);
+ 
+     if (slow_path != NULL) {
+       __ branch_destination(slow_path->continuation());
+       set_in_conditional_code(false);
+     }
+ 
+     element = x;
+   }
+ 
+   if (x->should_profile()) {
+     profile_element_type(element, md, load_store);
+   }
  }
  
+ void LIRGenerator::do_Deoptimize(Deoptimize* x) {
+   // This happens only when a class X uses the withfield/defaultvalue bytecode
+   // to refer to an inline class V, where V has not yet been loaded/resolved.
+   // This is not a common case. Let's just deoptimize.
+   CodeEmitInfo* info = state_for(x, x->state_before());
+   CodeStub* stub = new DeoptimizeStub(new CodeEmitInfo(info),
+                                       Deoptimization::Reason_unloaded,
+                                       Deoptimization::Action_make_not_entrant);
+   __ jump(stub);
+   LIR_Opr reg = rlock_result(x, T_OBJECT);
+   __ move(LIR_OprFact::oopConst(NULL), reg);
+ }
  
  void LIRGenerator::do_NullCheck(NullCheck* x) {
    if (x->can_trap()) {
      LIRItem value(x->obj(), this);
      value.load_item();

@@ -2489,11 +2874,11 @@
    if (!do_null && !do_update) {
      return result;
    }
  
    ciKlass* exact_signature_k = NULL;
-   if (do_update) {
+   if (do_update && signature_at_call_k != NULL) {
      // Is the type from the signature exact (the only one possible)?
      exact_signature_k = signature_at_call_k->exact_klass();
      if (exact_signature_k == NULL) {
        exact_signature_k = comp->cha_exact_type(signature_at_call_k);
      } else {

@@ -2574,10 +2959,56 @@
        }
      }
    }
  }
  
+ void LIRGenerator::profile_flags(ciMethodData* md, ciProfileData* data, int flag, LIR_Condition condition) {
+   assert(md != NULL && data != NULL, "should have been initialized");
+   LIR_Opr mdp = new_register(T_METADATA);
+   __ metadata2reg(md->constant_encoding(), mdp);
+   LIR_Address* addr = new LIR_Address(mdp, md->byte_offset_of_slot(data, DataLayout::flags_offset()), T_BYTE);
+   LIR_Opr flags = new_register(T_INT);
+   __ move(addr, flags);
+   if (condition != lir_cond_always) {
+     LIR_Opr update = new_register(T_INT);
+     __ cmove(condition, LIR_OprFact::intConst(0), LIR_OprFact::intConst(flag), update, T_INT);
+   } else {
+     __ logical_or(flags, LIR_OprFact::intConst(flag), flags);
+   }
+   __ store(flags, addr);
+ }
+ 
+ void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ciArrayLoadStoreData* load_store) {
+   assert(compilation()->profile_array_accesses(), "array access profiling is disabled");
+   LabelObj* L_end = new LabelObj();
+   LIR_Opr tmp = new_register(T_METADATA);
+   __ check_null_free_array(array.result(), tmp);
+ 
+   profile_flags(md, load_store, ArrayLoadStoreData::null_free_array_byte_constant(), lir_cond_equal);
+ }
+ 
+ void LIRGenerator::profile_array_type(AccessIndexed* x, ciMethodData*& md, ciArrayLoadStoreData*& load_store) {
+   assert(compilation()->profile_array_accesses(), "array access profiling is disabled");
+   int bci = x->profiled_bci();
+   md = x->profiled_method()->method_data();
+   assert(md != NULL, "Sanity");
+   ciProfileData* data = md->bci_to_data(bci);
+   assert(data != NULL && data->is_ArrayLoadStoreData(), "incorrect profiling entry");
+   load_store = (ciArrayLoadStoreData*)data;
+   LIR_Opr mdp = LIR_OprFact::illegalOpr;
+   profile_type(md, md->byte_offset_of_slot(load_store, ArrayLoadStoreData::array_offset()), 0,
+                load_store->array()->type(), x->array(), mdp, true, NULL, NULL);
+ }
+ 
+ void LIRGenerator::profile_element_type(Value element, ciMethodData* md, ciArrayLoadStoreData* load_store) {
+   assert(compilation()->profile_array_accesses(), "array access profiling is disabled");
+   assert(md != NULL && load_store != NULL, "should have been initialized");
+   LIR_Opr mdp = LIR_OprFact::illegalOpr;
+   profile_type(md, md->byte_offset_of_slot(load_store, ArrayLoadStoreData::element_offset()), 0,
+                load_store->element()->type(), element, mdp, false, NULL, NULL);
+ }
+ 
  void LIRGenerator::do_Base(Base* x) {
    __ std_entry(LIR_OprFact::illegalOpr);
    // Emit moves from physical registers / stack slots to virtual registers
    CallingConvention* args = compilation()->frame_map()->incoming_arguments();
    IRScope* irScope = compilation()->hir()->top_scope();

@@ -2658,10 +3089,18 @@
    if (!method()->is_accessor()) { // Accessors do not have MDOs, so no counting.
      profile_parameters(x);
      CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, SynchronizationEntryBCI), NULL, false);
      increment_invocation_counter(info);
    }
+   if (method()->has_scalarized_args()) {
+     // Check if deoptimization was triggered (i.e. orig_pc was set) while buffering scalarized inline type arguments
+     // in the entry point (see comments in frame::deoptimize). If so, deoptimize only now that we have the right state.
+     CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, 0), NULL, false);
+     CodeStub* deopt_stub = new DeoptimizeStub(info, Deoptimization::Reason_none, Deoptimization::Action_none);
+     __ append(new LIR_Op0(lir_check_orig_pc));
+     __ branch(lir_cond_notEqual, deopt_stub);
+   }
  
    // all blocks with a successor must end with an unconditional jump
    // to the successor even if they are consecutive
    __ jump(x->default_sux());
  }

@@ -2673,27 +3112,32 @@
    __ osr_entry(LIR_Assembler::osrBufferPointer());
    LIR_Opr result = rlock_result(x);
    __ move(LIR_Assembler::osrBufferPointer(), result);
  }
  
+ void LIRGenerator::invoke_load_one_argument(LIRItem* param, LIR_Opr loc) {
+   if (loc->is_register()) {
+     param->load_item_force(loc);
+   } else {
+     LIR_Address* addr = loc->as_address_ptr();
+     param->load_for_store(addr->type());
+     assert(addr->type() != T_INLINE_TYPE, "not supported yet");
+     if (addr->type() == T_OBJECT) {
+       __ move_wide(param->result(), addr);
+     } else {
+       __ move(param->result(), addr);
+     }
+   }
+ }
  
  void LIRGenerator::invoke_load_arguments(Invoke* x, LIRItemList* args, const LIR_OprList* arg_list) {
    assert(args->length() == arg_list->length(),
           "args=%d, arg_list=%d", args->length(), arg_list->length());
    for (int i = x->has_receiver() ? 1 : 0; i < args->length(); i++) {
      LIRItem* param = args->at(i);
      LIR_Opr loc = arg_list->at(i);
-     if (loc->is_register()) {
-       param->load_item_force(loc);
-     } else {
-       LIR_Address* addr = loc->as_address_ptr();
-       param->load_for_store(addr->type());
-       if (addr->type() == T_OBJECT) {
-         __ move_wide(param->result(), addr);
-       } else
-         __ move(param->result(), addr);
-     }
+     invoke_load_one_argument(param, loc);
    }
  
    if (x->has_receiver()) {
      LIRItem* receiver = args->at(0);
      LIR_Opr loc = arg_list->at(0);

@@ -2855,24 +3299,82 @@
  #endif
  
    LIRItem left(x->x(), this);
    LIRItem right(x->y(), this);
    left.load_item();
-   if (can_inline_as_constant(right.value())) {
+   if (can_inline_as_constant(right.value()) && !x->substitutability_check()) {
      right.dont_load_item();
    } else {
+     // substitutability_check() needs to use right as a base register.
      right.load_item();
    }
  
    LIRItem t_val(x->tval(), this);
    LIRItem f_val(x->fval(), this);
    t_val.dont_load_item();
    f_val.dont_load_item();
-   LIR_Opr reg = rlock_result(x);
  
-   __ cmp(lir_cond(x->cond()), left.result(), right.result());
-   __ cmove(lir_cond(x->cond()), t_val.result(), f_val.result(), reg, as_BasicType(x->x()->type()));
+   if (x->substitutability_check()) {
+     substitutability_check(x, left, right, t_val, f_val);
+   } else {
+     LIR_Opr reg = rlock_result(x);
+     __ cmp(lir_cond(x->cond()), left.result(), right.result());
+     __ cmove(lir_cond(x->cond()), t_val.result(), f_val.result(), reg, as_BasicType(x->x()->type()));
+   }
+ }
+ 
+ void LIRGenerator::substitutability_check(IfOp* x, LIRItem& left, LIRItem& right, LIRItem& t_val, LIRItem& f_val) {
+   assert(x->cond() == If::eql || x->cond() == If::neq, "must be");
+   bool is_acmpeq = (x->cond() == If::eql);
+   LIR_Opr equal_result     = is_acmpeq ? t_val.result() : f_val.result();
+   LIR_Opr not_equal_result = is_acmpeq ? f_val.result() : t_val.result();
+   LIR_Opr result = rlock_result(x);
+   CodeEmitInfo* info = state_for(x, x->state_before());
+ 
+   substitutability_check_common(x->x(), x->y(), left, right, equal_result, not_equal_result, result, info);
+ }
+ 
+ void LIRGenerator::substitutability_check(If* x, LIRItem& left, LIRItem& right) {
+   LIR_Opr equal_result     = LIR_OprFact::intConst(1);
+   LIR_Opr not_equal_result = LIR_OprFact::intConst(0);
+   LIR_Opr result = new_register(T_INT);
+   CodeEmitInfo* info = state_for(x, x->state_before());
+ 
+   substitutability_check_common(x->x(), x->y(), left, right, equal_result, not_equal_result, result, info);
+ 
+   assert(x->cond() == If::eql || x->cond() == If::neq, "must be");
+   __ cmp(lir_cond(x->cond()), result, equal_result);
+ }
+ 
+ void LIRGenerator::substitutability_check_common(Value left_val, Value right_val, LIRItem& left, LIRItem& right,
+                                                  LIR_Opr equal_result, LIR_Opr not_equal_result, LIR_Opr result,
+                                                  CodeEmitInfo* info) {
+   LIR_Opr tmp1 = LIR_OprFact::illegalOpr;
+   LIR_Opr tmp2 = LIR_OprFact::illegalOpr;
+   LIR_Opr left_klass_op = LIR_OprFact::illegalOpr;
+   LIR_Opr right_klass_op = LIR_OprFact::illegalOpr;
+ 
+   ciKlass* left_klass  = left_val ->as_loaded_klass_or_null();
+   ciKlass* right_klass = right_val->as_loaded_klass_or_null();
+ 
+   if ((left_klass == NULL || right_klass == NULL) ||// The klass is still unloaded, or came from a Phi node.
+       !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
+     init_temps_for_substitutability_check(tmp1, tmp2);
+   }
+ 
+   if (left_klass != NULL && left_klass->is_inlinetype() && left_klass == right_klass) {
+     // No need to load klass -- the operands are statically known to be the same inline klass.
+   } else {
+     BasicType t_klass = UseCompressedOops ? T_INT : T_METADATA;
+     left_klass_op = new_register(t_klass);
+     right_klass_op = new_register(t_klass);
+   }
+ 
+   CodeStub* slow_path = new SubstitutabilityCheckStub(left.result(), right.result(), info);
+   __ substitutability_check(result, left.result(), right.result(), equal_result, not_equal_result,
+                             tmp1, tmp2,
+                             left_klass, right_klass, left_klass_op, right_klass_op, info, slow_path);
  }
  
  #ifdef JFR_HAVE_INTRINSICS
  
  void LIRGenerator::do_getEventWriter(Intrinsic* x) {

@@ -3160,11 +3662,11 @@
    ciMethodData* md = x->method()->method_data_or_null();
    assert(md != NULL, "Sanity");
    ciProfileData* data = md->bci_to_data(bci);
    if (data != NULL) {
      assert(data->is_CallTypeData() || data->is_VirtualCallTypeData(), "wrong profile data type");
-     ciReturnTypeEntry* ret = data->is_CallTypeData() ? ((ciCallTypeData*)data)->ret() : ((ciVirtualCallTypeData*)data)->ret();
+     ciSingleTypeEntry* ret = data->is_CallTypeData() ? ((ciCallTypeData*)data)->ret() : ((ciVirtualCallTypeData*)data)->ret();
      LIR_Opr mdp = LIR_OprFact::illegalOpr;
  
      bool ignored_will_link;
      ciSignature* signature_at_call = NULL;
      x->method()->get_method_at_bci(bci, ignored_will_link, &signature_at_call);

@@ -3181,10 +3683,57 @@
        md->set_return_type(bci, exact);
      }
    }
  }
  
+ bool LIRGenerator::profile_inline_klass(ciMethodData* md, ciProfileData* data, Value value, int flag) {
+   ciKlass* klass = value->as_loaded_klass_or_null();
+   if (klass != NULL) {
+     if (klass->is_inlinetype()) {
+       profile_flags(md, data, flag, lir_cond_always);
+     } else if (klass->can_be_inline_klass()) {
+       return false;
+     }
+   } else {
+     return false;
+   }
+   return true;
+ }
+ 
+ 
+ void LIRGenerator::do_ProfileACmpTypes(ProfileACmpTypes* x) {
+   ciMethod* method = x->method();
+   assert(method != NULL, "method should be set if branch is profiled");
+   ciMethodData* md = method->method_data_or_null();
+   assert(md != NULL, "Sanity");
+   ciProfileData* data = md->bci_to_data(x->bci());
+   assert(data != NULL, "must have profiling data");
+   assert(data->is_ACmpData(), "need BranchData for two-way branches");
+   ciACmpData* acmp = (ciACmpData*)data;
+   LIR_Opr mdp = LIR_OprFact::illegalOpr;
+   profile_type(md, md->byte_offset_of_slot(acmp, ACmpData::left_offset()), 0,
+                acmp->left()->type(), x->left(), mdp, !x->left_maybe_null(), NULL, NULL);
+   int flags_offset = md->byte_offset_of_slot(data, DataLayout::flags_offset());
+   if (!profile_inline_klass(md, acmp, x->left(), ACmpData::left_inline_type_byte_constant())) {
+     LIR_Opr mdp = new_register(T_METADATA);
+     __ metadata2reg(md->constant_encoding(), mdp);
+     LIRItem value(x->left(), this);
+     value.load_item();
+     __ profile_inline_type(new LIR_Address(mdp, flags_offset, T_INT), value.result(), ACmpData::left_inline_type_byte_constant(), new_register(T_INT), !x->left_maybe_null());
+   }
+   profile_type(md, md->byte_offset_of_slot(acmp, ACmpData::left_offset()),
+                in_bytes(ACmpData::right_offset()) - in_bytes(ACmpData::left_offset()),
+                acmp->right()->type(), x->right(), mdp, !x->right_maybe_null(), NULL, NULL);
+   if (!profile_inline_klass(md, acmp, x->right(), ACmpData::right_inline_type_byte_constant())) {
+     LIR_Opr mdp = new_register(T_METADATA);
+     __ metadata2reg(md->constant_encoding(), mdp);
+     LIRItem value(x->right(), this);
+     value.load_item();
+     __ profile_inline_type(new LIR_Address(mdp, flags_offset, T_INT), value.result(), ACmpData::right_inline_type_byte_constant(), new_register(T_INT), !x->left_maybe_null());
+   }
+ }
+ 
  void LIRGenerator::do_ProfileInvoke(ProfileInvoke* x) {
    // We can safely ignore accessors here, since c2 will inline them anyway,
    // accessors are also always mature.
    if (!x->inlinee()->is_accessor()) {
      CodeEmitInfo* info = state_for(x, x->state(), true);
< prev index next >