< prev index next >

src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp

Print this page
@@ -26,10 +26,11 @@
  #ifndef _WINDOWS
  #include "alloca.h"
  #endif
  #include "asm/macroAssembler.hpp"
  #include "asm/macroAssembler.inline.hpp"
+ #include "classfile/symbolTable.hpp"
  #include "code/debugInfoRec.hpp"
  #include "code/icBuffer.hpp"
  #include "code/nativeInst.hpp"
  #include "code/vtableStubs.hpp"
  #include "compiler/oopMap.hpp"

@@ -526,10 +527,11 @@
        assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
        // fall through
      case T_OBJECT:
      case T_ARRAY:
      case T_ADDRESS:
+     case T_INLINE_TYPE:
        if (int_args < Argument::n_int_register_parameters_j) {
          regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
        } else {
          regs[i].set2(VMRegImpl::stack2reg(stk_args));
          stk_args += 2;

@@ -559,10 +561,92 @@
    }
  
    return align_up(stk_args, 2);
  }
  
+ // Same as java_calling_convention() but for multiple return
+ // values. There's no way to store them on the stack so if we don't
+ // have enough registers, multiple values can't be returned.
+ const uint SharedRuntime::java_return_convention_max_int = Argument::n_int_register_parameters_j+1;
+ const uint SharedRuntime::java_return_convention_max_float = Argument::n_float_register_parameters_j;
+ int SharedRuntime::java_return_convention(const BasicType *sig_bt,
+                                           VMRegPair *regs,
+                                           int total_args_passed) {
+   // Create the mapping between argument positions and
+   // registers.
+   static const Register INT_ArgReg[java_return_convention_max_int] = {
+     rax, j_rarg5, j_rarg4, j_rarg3, j_rarg2, j_rarg1, j_rarg0
+   };
+   static const XMMRegister FP_ArgReg[java_return_convention_max_float] = {
+     j_farg0, j_farg1, j_farg2, j_farg3,
+     j_farg4, j_farg5, j_farg6, j_farg7
+   };
+ 
+ 
+   uint int_args = 0;
+   uint fp_args = 0;
+ 
+   for (int i = 0; i < total_args_passed; i++) {
+     switch (sig_bt[i]) {
+     case T_BOOLEAN:
+     case T_CHAR:
+     case T_BYTE:
+     case T_SHORT:
+     case T_INT:
+       if (int_args < Argument::n_int_register_parameters_j+1) {
+         regs[i].set1(INT_ArgReg[int_args]->as_VMReg());
+         int_args++;
+       } else {
+         return -1;
+       }
+       break;
+     case T_VOID:
+       // halves of T_LONG or T_DOUBLE
+       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
+       regs[i].set_bad();
+       break;
+     case T_LONG:
+       assert(sig_bt[i + 1] == T_VOID, "expecting half");
+       // fall through
+     case T_OBJECT:
+     case T_INLINE_TYPE:
+     case T_ARRAY:
+     case T_ADDRESS:
+     case T_METADATA:
+       if (int_args < Argument::n_int_register_parameters_j+1) {
+         regs[i].set2(INT_ArgReg[int_args]->as_VMReg());
+         int_args++;
+       } else {
+         return -1;
+       }
+       break;
+     case T_FLOAT:
+       if (fp_args < Argument::n_float_register_parameters_j) {
+         regs[i].set1(FP_ArgReg[fp_args]->as_VMReg());
+         fp_args++;
+       } else {
+         return -1;
+       }
+       break;
+     case T_DOUBLE:
+       assert(sig_bt[i + 1] == T_VOID, "expecting half");
+       if (fp_args < Argument::n_float_register_parameters_j) {
+         regs[i].set2(FP_ArgReg[fp_args]->as_VMReg());
+         fp_args++;
+       } else {
+         return -1;
+       }
+       break;
+     default:
+       ShouldNotReachHere();
+       break;
+     }
+   }
+ 
+   return int_args + fp_args;
+ }
+ 
  // Patch the callers callsite with entry to compiled code if it exists.
  static void patch_callers_callsite(MacroAssembler *masm) {
    Label L;
    __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), (int32_t)NULL_WORD);
    __ jcc(Assembler::equal, L);

@@ -601,31 +685,182 @@
    // restore sp
    __ mov(rsp, r13);
    __ bind(L);
  }
  
+ // For each inline type argument, sig includes the list of fields of
+ // the inline type. This utility function computes the number of
+ // arguments for the call if inline types are passed by reference (the
+ // calling convention the interpreter expects).
+ static int compute_total_args_passed_int(const GrowableArray<SigEntry>* sig_extended) {
+   int total_args_passed = 0;
+   if (InlineTypePassFieldsAsArgs) {
+     for (int i = 0; i < sig_extended->length(); i++) {
+       BasicType bt = sig_extended->at(i)._bt;
+       if (bt == T_INLINE_TYPE) {
+         // In sig_extended, an inline type argument starts with:
+         // T_INLINE_TYPE, followed by the types of the fields of the
+         // inline type and T_VOID to mark the end of the value
+         // type. Inline types are flattened so, for instance, in the
+         // case of an inline type with an int field and an inline type
+         // field that itself has 2 fields, an int and a long:
+         // T_INLINE_TYPE T_INT T_INLINE_TYPE T_INT T_LONG T_VOID (second
+         // slot for the T_LONG) T_VOID (inner T_INLINE_TYPE) T_VOID
+         // (outer T_INLINE_TYPE)
+         total_args_passed++;
+         int vt = 1;
+         do {
+           i++;
+           BasicType bt = sig_extended->at(i)._bt;
+           BasicType prev_bt = sig_extended->at(i-1)._bt;
+           if (bt == T_INLINE_TYPE) {
+             vt++;
+           } else if (bt == T_VOID &&
+                      prev_bt != T_LONG &&
+                      prev_bt != T_DOUBLE) {
+             vt--;
+           }
+         } while (vt != 0);
+       } else {
+         total_args_passed++;
+       }
+     }
+   } else {
+     total_args_passed = sig_extended->length();
+   }
+   return total_args_passed;
+ }
+ 
+ 
+ static void gen_c2i_adapter_helper(MacroAssembler* masm,
+                                    BasicType bt,
+                                    BasicType prev_bt,
+                                    size_t size_in_bytes,
+                                    const VMRegPair& reg_pair,
+                                    const Address& to,
+                                    int extraspace,
+                                    bool is_oop) {
+   assert(bt != T_INLINE_TYPE || !InlineTypePassFieldsAsArgs, "no inline type here");
+   if (bt == T_VOID) {
+     assert(prev_bt == T_LONG || prev_bt == T_DOUBLE, "missing half");
+     return;
+   }
+ 
+   // Say 4 args:
+   // i   st_off
+   // 0   32 T_LONG
+   // 1   24 T_VOID
+   // 2   16 T_OBJECT
+   // 3    8 T_BOOL
+   // -    0 return address
+   //
+   // However to make thing extra confusing. Because we can fit a long/double in
+   // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
+   // leaves one slot empty and only stores to a single slot. In this case the
+   // slot that is occupied is the T_VOID slot. See I said it was confusing.
+ 
+   bool wide = (size_in_bytes == wordSize);
+   VMReg r_1 = reg_pair.first();
+   VMReg r_2 = reg_pair.second();
+   assert(r_2->is_valid() == wide, "invalid size");
+   if (!r_1->is_valid()) {
+     assert(!r_2->is_valid(), "must be invalid");
+     return;
+   }
+ 
+   if (!r_1->is_XMMRegister()) {
+     Register val = rax;
+     if (r_1->is_stack()) {
+       int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
+       __ load_sized_value(val, Address(rsp, ld_off), size_in_bytes, /* is_signed */ false);
+     } else {
+       val = r_1->as_Register();
+     }
+     assert_different_registers(to.base(), val, rscratch1);
+     if (is_oop) {
+       __ push(r13);
+       __ push(rbx);
+       __ store_heap_oop(to, val, rscratch1, r13, rbx, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
+       __ pop(rbx);
+       __ pop(r13);
+     } else {
+       __ store_sized_value(to, val, size_in_bytes);
+     }
+   } else {
+     if (wide) {
+       __ movdbl(to, r_1->as_XMMRegister());
+     } else {
+       __ movflt(to, r_1->as_XMMRegister());
+     }
+   }
+ }
  
  static void gen_c2i_adapter(MacroAssembler *masm,
-                             int total_args_passed,
-                             int comp_args_on_stack,
-                             const BasicType *sig_bt,
+                             const GrowableArray<SigEntry>* sig_extended,
                              const VMRegPair *regs,
-                             Label& skip_fixup) {
+                             Label& skip_fixup,
+                             address start,
+                             OopMapSet* oop_maps,
+                             int& frame_complete,
+                             int& frame_size_in_words,
+                             bool alloc_inline_receiver) {
    // Before we get into the guts of the C2I adapter, see if we should be here
    // at all.  We've come from compiled code and are attempting to jump to the
    // interpreter, which means the caller made a static call to get here
    // (vcalls always get a compiled target if there is one).  Check for a
    // compiled target.  If there is one, we need to patch the caller's call.
    patch_callers_callsite(masm);
  
    __ bind(skip_fixup);
  
+   if (InlineTypePassFieldsAsArgs) {
+     // Is there an inline type argument?
+     bool has_inline_argument = false;
+     for (int i = 0; i < sig_extended->length() && !has_inline_argument; i++) {
+       has_inline_argument = (sig_extended->at(i)._bt == T_INLINE_TYPE);
+     }
+     if (has_inline_argument) {
+       // There is at least an inline type argument: we're coming from
+       // compiled code so we have no buffers to back the inline types.
+       // Allocate the buffers here with a runtime call.
+       OopMap* map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_vectors*/ false);
+ 
+       frame_complete = __ offset();
+ 
+       __ set_last_Java_frame(noreg, noreg, NULL);
+ 
+       __ mov(c_rarg0, r15_thread);
+       __ mov(c_rarg1, rbx);
+       __ mov64(c_rarg2, (int64_t)alloc_inline_receiver);
+       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::allocate_inline_types)));
+ 
+       oop_maps->add_gc_map((int)(__ pc() - start), map);
+       __ reset_last_Java_frame(false);
+ 
+       RegisterSaver::restore_live_registers(masm);
+ 
+       Label no_exception;
+       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
+       __ jcc(Assembler::equal, no_exception);
+ 
+       __ movptr(Address(r15_thread, JavaThread::vm_result_offset()), (int)NULL_WORD);
+       __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
+       __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
+ 
+       __ bind(no_exception);
+ 
+       // We get an array of objects from the runtime call
+       __ get_vm_result(rscratch2, r15_thread); // Use rscratch2 (r11) as temporary because rscratch1 (r10) is trashed by movptr()
+       __ get_vm_result_2(rbx, r15_thread); // TODO: required to keep the callee Method live?
+     }
+   }
+ 
    // Since all args are passed on the stack, total_args_passed *
    // Interpreter::stackElementSize is the space we need. Plus 1 because
    // we also account for the return address location since
    // we store it first rather than hold it in rax across all the shuffling
- 
+   int total_args_passed = compute_total_args_passed_int(sig_extended);
    int extraspace = (total_args_passed * Interpreter::stackElementSize) + wordSize;
  
    // stack is aligned, keep it that way
    extraspace = align_up(extraspace, 2*wordSize);
  

@@ -639,100 +874,78 @@
  
    // Store the return address in the expected location
    __ movptr(Address(rsp, 0), rax);
  
    // Now write the args into the outgoing interpreter space
-   for (int i = 0; i < total_args_passed; i++) {
-     if (sig_bt[i] == T_VOID) {
-       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
-       continue;
-     }
- 
-     // offset to start parameters
-     int st_off   = (total_args_passed - i) * Interpreter::stackElementSize;
-     int next_off = st_off - Interpreter::stackElementSize;
- 
-     // Say 4 args:
-     // i   st_off
-     // 0   32 T_LONG
-     // 1   24 T_VOID
-     // 2   16 T_OBJECT
-     // 3    8 T_BOOL
-     // -    0 return address
-     //
-     // However to make thing extra confusing. Because we can fit a long/double in
-     // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
-     // leaves one slot empty and only stores to a single slot. In this case the
-     // slot that is occupied is the T_VOID slot. See I said it was confusing.
  
-     VMReg r_1 = regs[i].first();
-     VMReg r_2 = regs[i].second();
-     if (!r_1->is_valid()) {
-       assert(!r_2->is_valid(), "");
-       continue;
-     }
-     if (r_1->is_stack()) {
-       // memory to memory use rax
-       int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
-       if (!r_2->is_valid()) {
-         // sign extend??
-         __ movl(rax, Address(rsp, ld_off));
-         __ movptr(Address(rsp, st_off), rax);
- 
-       } else {
- 
-         __ movq(rax, Address(rsp, ld_off));
- 
-         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
-         // T_DOUBLE and T_LONG use two slots in the interpreter
-         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
-           // ld_off == LSW, ld_off+wordSize == MSW
-           // st_off == MSW, next_off == LSW
-           __ movq(Address(rsp, next_off), rax);
+   // next_arg_comp is the next argument from the compiler point of
+   // view (inline type fields are passed in registers/on the stack). In
+   // sig_extended, an inline type argument starts with: T_INLINE_TYPE,
+   // followed by the types of the fields of the inline type and T_VOID
+   // to mark the end of the inline type. ignored counts the number of
+   // T_INLINE_TYPE/T_VOID. next_vt_arg is the next inline type argument:
+   // used to get the buffer for that argument from the pool of buffers
+   // we allocated above and want to pass to the
+   // interpreter. next_arg_int is the next argument from the
+   // interpreter point of view (inline types are passed by reference).
+   for (int next_arg_comp = 0, ignored = 0, next_vt_arg = 0, next_arg_int = 0;
+        next_arg_comp < sig_extended->length(); next_arg_comp++) {
+     assert(ignored <= next_arg_comp, "shouldn't skip over more slots than there are arguments");
+     assert(next_arg_int <= total_args_passed, "more arguments for the interpreter than expected?");
+     BasicType bt = sig_extended->at(next_arg_comp)._bt;
+     int st_off = (total_args_passed - next_arg_int) * Interpreter::stackElementSize;
+     if (!InlineTypePassFieldsAsArgs || bt != T_INLINE_TYPE) {
+       int next_off = st_off - Interpreter::stackElementSize;
+       const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : st_off;
+       const VMRegPair reg_pair = regs[next_arg_comp-ignored];
+       size_t size_in_bytes = reg_pair.second()->is_valid() ? 8 : 4;
+       gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
+                              size_in_bytes, reg_pair, Address(rsp, offset), extraspace, false);
+       next_arg_int++;
  #ifdef ASSERT
-           // Overwrite the unused slot with known junk
-           __ mov64(rax, CONST64(0xdeadffffdeadaaaa));
-           __ movptr(Address(rsp, st_off), rax);
- #endif /* ASSERT */
-         } else {
-           __ movq(Address(rsp, st_off), rax);
-         }
+       if (bt == T_LONG || bt == T_DOUBLE) {
+         // Overwrite the unused slot with known junk
+         __ mov64(rax, CONST64(0xdeadffffdeadaaaa));
+         __ movptr(Address(rsp, st_off), rax);
        }
-     } else if (r_1->is_Register()) {
-       Register r = r_1->as_Register();
-       if (!r_2->is_valid()) {
-         // must be only an int (or less ) so move only 32bits to slot
-         // why not sign extend??
-         __ movl(Address(rsp, st_off), r);
-       } else {
-         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
-         // T_DOUBLE and T_LONG use two slots in the interpreter
-         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
-           // long/double in gpr
- #ifdef ASSERT
-           // Overwrite the unused slot with known junk
-           __ mov64(rax, CONST64(0xdeadffffdeadaaab));
-           __ movptr(Address(rsp, st_off), rax);
  #endif /* ASSERT */
-           __ movq(Address(rsp, next_off), r);
+     } else {
+       ignored++;
+       // get the buffer from the just allocated pool of buffers
+       int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + next_vt_arg * type2aelembytes(T_INLINE_TYPE);
+       __ load_heap_oop(r14, Address(rscratch2, index));
+       next_vt_arg++; next_arg_int++;
+       int vt = 1;
+       // write fields we get from compiled code in registers/stack
+       // slots to the buffer: we know we are done with that inline type
+       // argument when we hit the T_VOID that acts as an end of inline
+       // type delimiter for this inline type. Inline types are flattened
+       // so we might encounter embedded inline types. Each entry in
+       // sig_extended contains a field offset in the buffer.
+       do {
+         next_arg_comp++;
+         BasicType bt = sig_extended->at(next_arg_comp)._bt;
+         BasicType prev_bt = sig_extended->at(next_arg_comp-1)._bt;
+         if (bt == T_INLINE_TYPE) {
+           vt++;
+           ignored++;
+         } else if (bt == T_VOID &&
+                    prev_bt != T_LONG &&
+                    prev_bt != T_DOUBLE) {
+           vt--;
+           ignored++;
          } else {
-           __ movptr(Address(rsp, st_off), r);
+           int off = sig_extended->at(next_arg_comp)._offset;
+           assert(off > 0, "offset in object should be positive");
+           size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
+           bool is_oop = is_reference_type(bt);
+           gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
+                                  size_in_bytes, regs[next_arg_comp-ignored], Address(r14, off), extraspace, is_oop);
          }
-       }
-     } else {
-       assert(r_1->is_XMMRegister(), "");
-       if (!r_2->is_valid()) {
-         // only a float use just part of the slot
-         __ movflt(Address(rsp, st_off), r_1->as_XMMRegister());
-       } else {
- #ifdef ASSERT
-         // Overwrite the unused slot with known junk
-         __ mov64(rax, CONST64(0xdeadffffdeadaaac));
-         __ movptr(Address(rsp, st_off), rax);
- #endif /* ASSERT */
-         __ movdbl(Address(rsp, next_off), r_1->as_XMMRegister());
-       }
+       } while (vt != 0);
+       // pass the buffer to the interpreter
+       __ movptr(Address(rsp, st_off), r14);
      }
    }
  
    // Schedule the branch target address early.
    __ movptr(rcx, Address(rbx, in_bytes(Method::interpreter_entry_offset())));

@@ -751,13 +964,12 @@
    __ jcc(Assembler::below, L_ok);
    __ bind(L_fail);
  }
  
  void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
-                                     int total_args_passed,
                                      int comp_args_on_stack,
-                                     const BasicType *sig_bt,
+                                     const GrowableArray<SigEntry>* sig,
                                      const VMRegPair *regs) {
  
    // Note: r13 contains the senderSP on entry. We must preserve it since
    // we may do a i2c -> c2i transition if we lose a race where compiled
    // code goes non-entrant while we get args ready.

@@ -845,11 +1057,11 @@
    const Register saved_sp = rax;
    __ movptr(saved_sp, r11);
  
    // Will jump to the compiled code just as if compiled code was doing it.
    // Pre-load the register-jump target early, to schedule it better.
-   __ movptr(r11, Address(rbx, in_bytes(Method::from_compiled_offset())));
+   __ movptr(r11, Address(rbx, in_bytes(Method::from_compiled_inline_offset())));
  
  #if INCLUDE_JVMCI
    if (EnableJVMCI) {
      // check if this call should be routed towards a specific entry point
      __ cmpptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);

@@ -859,17 +1071,22 @@
      __ movptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);
      __ bind(no_alternative_target);
    }
  #endif // INCLUDE_JVMCI
  
+   int total_args_passed = sig->length();
+ 
    // Now generate the shuffle code.  Pick up all register args and move the
    // rest through the floating point stack top.
    for (int i = 0; i < total_args_passed; i++) {
-     if (sig_bt[i] == T_VOID) {
+     BasicType bt = sig->at(i)._bt;
+     assert(bt != T_INLINE_TYPE, "i2c adapter doesn't unpack inline type args");
+     if (bt == T_VOID) {
        // Longs and doubles are passed in native word order, but misaligned
        // in the 32-bit build.
-       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
+       BasicType prev_bt = (i > 0) ? sig->at(i-1)._bt : T_ILLEGAL;
+       assert(i > 0 && (prev_bt == T_LONG || prev_bt == T_DOUBLE), "missing half");
        continue;
      }
  
      // Pick up 0, 1 or 2 words from SP+offset.
  

@@ -907,11 +1124,11 @@
          //
          // Interpreter local[n] == MSW, local[n+1] == LSW however locals
          // are accessed as negative so LSW is at LOW address
  
          // ld_off is MSW so get LSW
-         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
+         const int offset = (bt==T_LONG||bt==T_DOUBLE)?
                             next_off : ld_off;
          __ movq(r13, Address(saved_sp, offset));
          // st_off is LSW (i.e. reg.first())
          __ movq(Address(rsp, st_off), r13);
        }

@@ -922,11 +1139,11 @@
          //
          // We are using two VMRegs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE
          // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case
          // So we must adjust where to pick up the data to match the interpreter.
  
-         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
+         const int offset = (bt==T_LONG||bt==T_DOUBLE)?
                             next_off : ld_off;
  
          // this can be a misaligned move
          __ movq(r, Address(saved_sp, offset));
        } else {

@@ -953,26 +1170,52 @@
    // and the vm will find there should this case occur.
  
    __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx);
  
    // put Method* where a c2i would expect should we end up there
-   // only needed becaus eof c2 resolve stubs return Method* as a result in
+   // only needed because of c2 resolve stubs return Method* as a result in
    // rax
    __ mov(rax, rbx);
    __ jmp(r11);
  }
  
+ static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) {
+   Label ok;
+ 
+   Register holder = rax;
+   Register receiver = j_rarg0;
+   Register temp = rbx;
+ 
+   __ load_klass(temp, receiver, rscratch1);
+   __ cmpptr(temp, Address(holder, CompiledICHolder::holder_klass_offset()));
+   __ movptr(rbx, Address(holder, CompiledICHolder::holder_metadata_offset()));
+   __ jcc(Assembler::equal, ok);
+   __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ 
+   __ bind(ok);
+   // Method might have been compiled since the call site was patched to
+   // interpreted if that is the case treat it as a miss so we can get
+   // the call site corrected.
+   __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), (int32_t)NULL_WORD);
+   __ jcc(Assembler::equal, skip_fixup);
+   __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ }
+ 
  // ---------------------------------------------------------------
- AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
-                                                             int total_args_passed,
+ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,
                                                              int comp_args_on_stack,
-                                                             const BasicType *sig_bt,
-                                                             const VMRegPair *regs,
-                                                             AdapterFingerPrint* fingerprint) {
+                                                             const GrowableArray<SigEntry>* sig,
+                                                             const VMRegPair* regs,
+                                                             const GrowableArray<SigEntry>* sig_cc,
+                                                             const VMRegPair* regs_cc,
+                                                             const GrowableArray<SigEntry>* sig_cc_ro,
+                                                             const VMRegPair* regs_cc_ro,
+                                                             AdapterFingerPrint* fingerprint,
+                                                             AdapterBlob*& new_adapter,
+                                                             bool allocate_code_blob) {
    address i2c_entry = __ pc();
- 
-   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
+   gen_i2c_adapter(masm, comp_args_on_stack, sig, regs);
  
    // -------------------------------------------------------------------------
    // Generate a C2I adapter.  On entry we know rbx holds the Method* during calls
    // to the interpreter.  The args start out packed in the compiled layout.  They
    // need to be unpacked into the interpreter layout.  This will almost always

@@ -981,32 +1224,25 @@
    // On exit from the interpreter, the interpreter will restore our SP (lest the
    // compiled code, which relys solely on SP and not RBP, get sick).
  
    address c2i_unverified_entry = __ pc();
    Label skip_fixup;
-   Label ok;
- 
-   Register holder = rax;
-   Register receiver = j_rarg0;
-   Register temp = rbx;
  
-   {
-     __ load_klass(temp, receiver, rscratch1);
-     __ cmpptr(temp, Address(holder, CompiledICHolder::holder_klass_offset()));
-     __ movptr(rbx, Address(holder, CompiledICHolder::holder_metadata_offset()));
-     __ jcc(Assembler::equal, ok);
-     __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+   gen_inline_cache_check(masm, skip_fixup);
  
-     __ bind(ok);
-     // Method might have been compiled since the call site was patched to
-     // interpreted if that is the case treat it as a miss so we can get
-     // the call site corrected.
-     __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), (int32_t)NULL_WORD);
-     __ jcc(Assembler::equal, skip_fixup);
-     __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+   OopMapSet* oop_maps = new OopMapSet();
+   int frame_complete = CodeOffsets::frame_never_safe;
+   int frame_size_in_words = 0;
+ 
+   // Scalarized c2i adapter with non-scalarized receiver (i.e., don't pack receiver)
+   address c2i_inline_ro_entry = __ pc();
+   if (regs_cc != regs_cc_ro) {
+     gen_c2i_adapter(masm, sig_cc_ro, regs_cc_ro, skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, false);
+     skip_fixup.reset();
    }
  
+   // Scalarized c2i adapter
    address c2i_entry = __ pc();
  
    // Class initialization barrier for static methods
    address c2i_no_clinit_check_entry = NULL;
    if (VM_Version::supports_fast_class_init_checks()) {

@@ -1031,14 +1267,35 @@
    }
  
    BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
    bs->c2i_entry_barrier(masm);
  
-   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
+   gen_c2i_adapter(masm, sig_cc, regs_cc, skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, true);
+ 
+   address c2i_unverified_inline_entry = c2i_unverified_entry;
+ 
+   // Non-scalarized c2i adapter
+   address c2i_inline_entry = c2i_entry;
+   if (regs != regs_cc) {
+     Label inline_entry_skip_fixup;
+     c2i_unverified_inline_entry = __ pc();
+     gen_inline_cache_check(masm, inline_entry_skip_fixup);
+ 
+     c2i_inline_entry = __ pc();
+     gen_c2i_adapter(masm, sig, regs, inline_entry_skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, false);
+   }
  
    __ flush();
-   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
+ 
+   // The c2i adapters might safepoint and trigger a GC. The caller must make sure that
+   // the GC knows about the location of oop argument locations passed to the c2i adapter.
+   if (allocate_code_blob) {
+     bool caller_must_gc_arguments = (regs != regs_cc);
+     new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
+   }
+ 
+   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry, c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
  }
  
  int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
                                           VMRegPair *regs,
                                           VMRegPair *regs2,

@@ -1092,10 +1349,11 @@
        case T_LONG:
          assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
          // fall through
        case T_OBJECT:
        case T_ARRAY:
+       case T_INLINE_TYPE:
        case T_ADDRESS:
        case T_METADATA:
          if (int_args < Argument::n_int_register_parameters_c) {
            regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
  #ifdef _WIN64

@@ -1948,10 +2206,11 @@
              freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
            }
  #endif
            break;
          }
+       case T_INLINE_TYPE:
        case T_OBJECT:
          assert(!is_critical_native, "no oop arguments");
          __ object_move(map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
                      ((i == 0) && (!is_static)),
                      &receiver_offset);

@@ -2078,10 +2337,15 @@
      // Load immediate 1 into swap_reg %rax
      __ movl(swap_reg, 1);
  
      // Load (object->mark() | 1) into swap_reg %rax
      __ orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+     if (EnableValhalla) {
+       // Mask inline_type bit such that we go to the slow path if object is an inline type
+       __ andptr(swap_reg, ~((int) markWord::inline_type_bit_in_place));
+     }
+ 
  
      // Save (object->mark() | 1) into BasicLock's displaced header
      __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
  
      // src -> dest iff dest == rax else rax <- dest

@@ -2137,10 +2401,11 @@
    case T_DOUBLE :
    case T_FLOAT  :
      // Result is in xmm0 we'll save as needed
      break;
    case T_ARRAY:                 // Really a handle
+   case T_INLINE_TYPE:           // Really a handle
    case T_OBJECT:                // Really a handle
        break; // can't de-handlize until after safepoint check
    case T_VOID: break;
    case T_LONG: break;
    default       : ShouldNotReachHere();

@@ -3891,10 +4156,121 @@
    // Set exception blob
    _exception_blob =  ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
  }
  #endif // COMPILER2
  
+ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) {
+   BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K);
+   CodeBuffer buffer(buf);
+   short buffer_locs[20];
+   buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
+                                          sizeof(buffer_locs)/sizeof(relocInfo));
+ 
+   MacroAssembler* masm = new MacroAssembler(&buffer);
+ 
+   const Array<SigEntry>* sig_vk = vk->extended_sig();
+   const Array<VMRegPair>* regs = vk->return_regs();
+ 
+   int pack_fields_jobject_off = __ offset();
+   // Resolve pre-allocated buffer from JNI handle.
+   // We cannot do this in generate_call_stub() because it requires GC code to be initialized.
+   __ movptr(rax, Address(r13, 0));
+   __ resolve_jobject(rax /* value */,
+                      r15_thread /* thread */,
+                      r12 /* tmp */);
+   __ movptr(Address(r13, 0), rax);
+ 
+   int pack_fields_off = __ offset();
+ 
+   int j = 1;
+   for (int i = 0; i < sig_vk->length(); i++) {
+     BasicType bt = sig_vk->at(i)._bt;
+     if (bt == T_INLINE_TYPE) {
+       continue;
+     }
+     if (bt == T_VOID) {
+       if (sig_vk->at(i-1)._bt == T_LONG ||
+           sig_vk->at(i-1)._bt == T_DOUBLE) {
+         j++;
+       }
+       continue;
+     }
+     int off = sig_vk->at(i)._offset;
+     assert(off > 0, "offset in object should be positive");
+     VMRegPair pair = regs->at(j);
+     VMReg r_1 = pair.first();
+     VMReg r_2 = pair.second();
+     Address to(rax, off);
+     if (bt == T_FLOAT) {
+       __ movflt(to, r_1->as_XMMRegister());
+     } else if (bt == T_DOUBLE) {
+       __ movdbl(to, r_1->as_XMMRegister());
+     } else {
+       Register val = r_1->as_Register();
+       assert_different_registers(to.base(), val, r14, r13, rbx, rscratch1);
+       if (is_reference_type(bt)) {
+         __ store_heap_oop(to, val, r14, r13, rbx, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
+       } else {
+         __ store_sized_value(to, r_1->as_Register(), type2aelembytes(bt));
+       }
+     }
+     j++;
+   }
+   assert(j == regs->length(), "missed a field?");
+ 
+   __ ret(0);
+ 
+   int unpack_fields_off = __ offset();
+ 
+   j = 1;
+   for (int i = 0; i < sig_vk->length(); i++) {
+     BasicType bt = sig_vk->at(i)._bt;
+     if (bt == T_INLINE_TYPE) {
+       continue;
+     }
+     if (bt == T_VOID) {
+       if (sig_vk->at(i-1)._bt == T_LONG ||
+           sig_vk->at(i-1)._bt == T_DOUBLE) {
+         j++;
+       }
+       continue;
+     }
+     int off = sig_vk->at(i)._offset;
+     assert(off > 0, "offset in object should be positive");
+     VMRegPair pair = regs->at(j);
+     VMReg r_1 = pair.first();
+     VMReg r_2 = pair.second();
+     Address from(rax, off);
+     if (bt == T_FLOAT) {
+       __ movflt(r_1->as_XMMRegister(), from);
+     } else if (bt == T_DOUBLE) {
+       __ movdbl(r_1->as_XMMRegister(), from);
+     } else if (bt == T_OBJECT || bt == T_ARRAY) {
+       assert_different_registers(rax, r_1->as_Register());
+       __ load_heap_oop(r_1->as_Register(), from);
+     } else {
+       assert(is_java_primitive(bt), "unexpected basic type");
+       assert_different_registers(rax, r_1->as_Register());
+       size_t size_in_bytes = type2aelembytes(bt);
+       __ load_sized_value(r_1->as_Register(), from, size_in_bytes, bt != T_CHAR && bt != T_BOOLEAN);
+     }
+     j++;
+   }
+   assert(j == regs->length(), "missed a field?");
+ 
+   if (StressInlineTypeReturnedAsFields) {
+     __ load_klass(rax, rax, rscratch1);
+     __ orptr(rax, 1);
+   }
+ 
+   __ ret(0);
+ 
+   __ flush();
+ 
+   return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
+ }
+ 
  void SharedRuntime::compute_move_order(const BasicType* in_sig_bt,
                                         int total_in_args, const VMRegPair* in_regs,
                                         int total_out_args, VMRegPair* out_regs,
                                         GrowableArray<int>& arg_order,
                                         VMRegPair tmp_vmreg) {
< prev index next >