10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "code/codeCache.hpp"
31 #include "code/debugInfoRec.hpp"
32 #include "code/icBuffer.hpp"
33 #include "code/vtableStubs.hpp"
34 #include "compiler/oopMap.hpp"
35 #include "gc/shared/barrierSetAssembler.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "interpreter/interp_masm.hpp"
38 #include "logging/log.hpp"
39 #include "memory/resourceArea.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/compiledICHolder.hpp"
42 #include "oops/klass.inline.hpp"
43 #include "prims/methodHandles.hpp"
44 #include "runtime/jniHandles.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/signature.hpp"
48 #include "runtime/stubRoutines.hpp"
49 #include "runtime/vframeArray.hpp"
303 case T_SHORT:
304 case T_INT:
305 if (int_args < Argument::n_int_register_parameters_j) {
306 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
307 } else {
308 regs[i].set1(VMRegImpl::stack2reg(stk_args));
309 stk_args += 2;
310 }
311 break;
312 case T_VOID:
313 // halves of T_LONG or T_DOUBLE
314 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
315 regs[i].set_bad();
316 break;
317 case T_LONG:
318 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
319 // fall through
320 case T_OBJECT:
321 case T_ARRAY:
322 case T_ADDRESS:
323 if (int_args < Argument::n_int_register_parameters_j) {
324 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
325 } else {
326 regs[i].set2(VMRegImpl::stack2reg(stk_args));
327 stk_args += 2;
328 }
329 break;
330 case T_FLOAT:
331 if (fp_args < Argument::n_float_register_parameters_j) {
332 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
333 } else {
334 regs[i].set1(VMRegImpl::stack2reg(stk_args));
335 stk_args += 2;
336 }
337 break;
338 case T_DOUBLE:
339 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
340 if (fp_args < Argument::n_float_register_parameters_j) {
341 regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
342 } else {
343 regs[i].set2(VMRegImpl::stack2reg(stk_args));
344 stk_args += 2;
345 }
346 break;
347 default:
348 ShouldNotReachHere();
349 break;
350 }
351 }
352
353 return align_up(stk_args, 2);
354 }
355
356 // Patch the callers callsite with entry to compiled code if it exists.
357 static void patch_callers_callsite(MacroAssembler *masm) {
358 Label L;
359 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
360 __ cbz(rscratch1, L);
361
362 __ enter();
363 __ push_CPU_state();
364
365 // VM needs caller's callsite
366 // VM needs target method
367 // This needs to be a long call since we will relocate this adapter to
368 // the codeBuffer and it may not reach
369
370 #ifndef PRODUCT
371 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
372 #endif
373
374 __ mov(c_rarg0, rmethod);
375 __ mov(c_rarg1, lr);
376 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
377 __ blr(rscratch1);
378
379 // Explicit isb required because fixup_callers_callsite may change the code
380 // stream.
381 __ safepoint_isb();
382
383 __ pop_CPU_state();
384 // restore sp
385 __ leave();
386 __ bind(L);
387 }
388
389 static void gen_c2i_adapter(MacroAssembler *masm,
390 int total_args_passed,
391 int comp_args_on_stack,
392 const BasicType *sig_bt,
393 const VMRegPair *regs,
394 Label& skip_fixup) {
395 // Before we get into the guts of the C2I adapter, see if we should be here
396 // at all. We've come from compiled code and are attempting to jump to the
397 // interpreter, which means the caller made a static call to get here
398 // (vcalls always get a compiled target if there is one). Check for a
399 // compiled target. If there is one, we need to patch the caller's call.
400 patch_callers_callsite(masm);
401
402 __ bind(skip_fixup);
403
404 int words_pushed = 0;
405
406 // Since all args are passed on the stack, total_args_passed *
407 // Interpreter::stackElementSize is the space we need.
408
409 int extraspace = total_args_passed * Interpreter::stackElementSize;
410
411 __ mov(r13, sp);
412
413 // stack is aligned, keep it that way
414 extraspace = align_up(extraspace, 2*wordSize);
415
416 if (extraspace)
417 __ sub(sp, sp, extraspace);
418
419 // Now write the args into the outgoing interpreter space
420 for (int i = 0; i < total_args_passed; i++) {
421 if (sig_bt[i] == T_VOID) {
422 assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
423 continue;
424 }
425
426 // offset to start parameters
427 int st_off = (total_args_passed - i - 1) * Interpreter::stackElementSize;
428 int next_off = st_off - Interpreter::stackElementSize;
429
430 // Say 4 args:
431 // i st_off
432 // 0 32 T_LONG
433 // 1 24 T_VOID
434 // 2 16 T_OBJECT
435 // 3 8 T_BOOL
436 // - 0 return address
437 //
438 // However to make thing extra confusing. Because we can fit a Java long/double in
439 // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
440 // leaves one slot empty and only stores to a single slot. In this case the
441 // slot that is occupied is the T_VOID slot. See I said it was confusing.
442
443 VMReg r_1 = regs[i].first();
444 VMReg r_2 = regs[i].second();
445 if (!r_1->is_valid()) {
446 assert(!r_2->is_valid(), "");
447 continue;
448 }
449 if (r_1->is_stack()) {
450 // memory to memory use rscratch1
451 int ld_off = (r_1->reg2stack() * VMRegImpl::stack_slot_size
452 + extraspace
453 + words_pushed * wordSize);
454 if (!r_2->is_valid()) {
455 // sign extend??
456 __ ldrw(rscratch1, Address(sp, ld_off));
457 __ str(rscratch1, Address(sp, st_off));
458
459 } else {
460
461 __ ldr(rscratch1, Address(sp, ld_off));
462
463 // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
464 // T_DOUBLE and T_LONG use two slots in the interpreter
465 if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
466 // ld_off == LSW, ld_off+wordSize == MSW
467 // st_off == MSW, next_off == LSW
468 __ str(rscratch1, Address(sp, next_off));
469 #ifdef ASSERT
470 // Overwrite the unused slot with known junk
471 __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaaaull);
472 __ str(rscratch1, Address(sp, st_off));
473 #endif /* ASSERT */
474 } else {
475 __ str(rscratch1, Address(sp, st_off));
476 }
477 }
478 } else if (r_1->is_Register()) {
479 Register r = r_1->as_Register();
480 if (!r_2->is_valid()) {
481 // must be only an int (or less ) so move only 32bits to slot
482 // why not sign extend??
483 __ str(r, Address(sp, st_off));
484 } else {
485 // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
486 // T_DOUBLE and T_LONG use two slots in the interpreter
487 if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
488 // jlong/double in gpr
489 #ifdef ASSERT
490 // Overwrite the unused slot with known junk
491 __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaabull);
492 __ str(rscratch1, Address(sp, st_off));
493 #endif /* ASSERT */
494 __ str(r, Address(sp, next_off));
495 } else {
496 __ str(r, Address(sp, st_off));
497 }
498 }
499 } else {
500 assert(r_1->is_FloatRegister(), "");
501 if (!r_2->is_valid()) {
502 // only a float use just part of the slot
503 __ strs(r_1->as_FloatRegister(), Address(sp, st_off));
504 } else {
505 #ifdef ASSERT
506 // Overwrite the unused slot with known junk
507 __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaacull);
508 __ str(rscratch1, Address(sp, st_off));
509 #endif /* ASSERT */
510 __ strd(r_1->as_FloatRegister(), Address(sp, next_off));
511 }
512 }
513 }
514
515 __ mov(esp, sp); // Interp expects args on caller's expression stack
516
517 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
518 __ br(rscratch1);
519 }
520
521
522 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
523 int total_args_passed,
524 int comp_args_on_stack,
525 const BasicType *sig_bt,
526 const VMRegPair *regs) {
527
528 // Note: r13 contains the senderSP on entry. We must preserve it since
529 // we may do a i2c -> c2i transition if we lose a race where compiled
530 // code goes non-entrant while we get args ready.
531
532 // In addition we use r13 to locate all the interpreter args because
533 // we must align the stack to 16 bytes.
534
535 // Adapters are frameless.
536
537 // An i2c adapter is frameless because the *caller* frame, which is
538 // interpreted, routinely repairs its own esp (from
539 // interpreter_frame_last_sp), even if a callee has modified the
540 // stack pointer. It also recalculates and aligns sp.
541
542 // A c2i adapter is frameless because the *callee* frame, which is
543 // interpreted, routinely repairs its caller's sp (from sender_sp,
544 // which is set up via the senderSP register).
545
546 // In other words, if *either* the caller or callee is interpreted, we can
566 range_check(masm, rax, r11,
567 Interpreter::code()->code_start(), Interpreter::code()->code_end(),
568 L_ok);
569 if (StubRoutines::code1() != NULL)
570 range_check(masm, rax, r11,
571 StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
572 L_ok);
573 if (StubRoutines::code2() != NULL)
574 range_check(masm, rax, r11,
575 StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
576 L_ok);
577 const char* msg = "i2c adapter must return to an interpreter frame";
578 __ block_comment(msg);
579 __ stop(msg);
580 __ bind(L_ok);
581 __ block_comment("} verify_i2ce ");
582 #endif
583 }
584
585 // Cut-out for having no stack args.
586 int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
587 if (comp_args_on_stack) {
588 __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
589 __ andr(sp, rscratch1, -16);
590 }
591
592 // Will jump to the compiled code just as if compiled code was doing it.
593 // Pre-load the register-jump target early, to schedule it better.
594 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_offset())));
595
596 #if INCLUDE_JVMCI
597 if (EnableJVMCI) {
598 // check if this call should be routed towards a specific entry point
599 __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
600 Label no_alternative_target;
601 __ cbz(rscratch2, no_alternative_target);
602 __ mov(rscratch1, rscratch2);
603 __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
604 __ bind(no_alternative_target);
605 }
606 #endif // INCLUDE_JVMCI
607
608 // Now generate the shuffle code.
609 for (int i = 0; i < total_args_passed; i++) {
610 if (sig_bt[i] == T_VOID) {
611 assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
612 continue;
613 }
614
615 // Pick up 0, 1 or 2 words from SP+offset.
616
617 assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
618 "scrambled load targets?");
619 // Load in argument order going down.
620 int ld_off = (total_args_passed - i - 1)*Interpreter::stackElementSize;
621 // Point to interpreter value (vs. tag)
622 int next_off = ld_off - Interpreter::stackElementSize;
623 //
624 //
625 //
626 VMReg r_1 = regs[i].first();
627 VMReg r_2 = regs[i].second();
628 if (!r_1->is_valid()) {
629 assert(!r_2->is_valid(), "");
630 continue;
631 }
632 if (r_1->is_stack()) {
633 // Convert stack slot to an SP offset (+ wordSize to account for return address )
634 int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size;
635 if (!r_2->is_valid()) {
636 // sign extend???
637 __ ldrsw(rscratch2, Address(esp, ld_off));
638 __ str(rscratch2, Address(sp, st_off));
639 } else {
640 //
641 // We are using two optoregs. This can be either T_OBJECT,
642 // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
643 // two slots but only uses one for thr T_LONG or T_DOUBLE case
644 // So we must adjust where to pick up the data to match the
645 // interpreter.
646 //
647 // Interpreter local[n] == MSW, local[n+1] == LSW however locals
648 // are accessed as negative so LSW is at LOW address
649
650 // ld_off is MSW so get LSW
651 const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
652 next_off : ld_off;
653 __ ldr(rscratch2, Address(esp, offset));
654 // st_off is LSW (i.e. reg.first())
655 __ str(rscratch2, Address(sp, st_off));
656 }
657 } else if (r_1->is_Register()) { // Register argument
658 Register r = r_1->as_Register();
659 if (r_2->is_valid()) {
660 //
661 // We are using two VMRegs. This can be either T_OBJECT,
662 // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
663 // two slots but only uses one for thr T_LONG or T_DOUBLE case
664 // So we must adjust where to pick up the data to match the
665 // interpreter.
666
667 const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
668 next_off : ld_off;
669
670 // this can be a misaligned move
671 __ ldr(r, Address(esp, offset));
672 } else {
673 // sign extend and use a full word?
674 __ ldrw(r, Address(esp, ld_off));
675 }
676 } else {
677 if (!r_2->is_valid()) {
678 __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
679 } else {
680 __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
681 }
682 }
683 }
684
685 // 6243940 We might end up in handle_wrong_method if
686 // the callee is deoptimized as we race thru here. If that
687 // happens we don't want to take a safepoint because the
688 // caller frame will look interpreted and arguments are now
689 // "compiled" so it is much better to make this transition
690 // invisible to the stack walking code. Unfortunately if
691 // we try and find the callee by normal means a safepoint
692 // is possible. So we stash the desired callee in the thread
693 // and the vm will find there should this case occur.
694
695 __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));
696
697 __ br(rscratch1);
698 }
699
700 // ---------------------------------------------------------------
701 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
702 int total_args_passed,
703 int comp_args_on_stack,
704 const BasicType *sig_bt,
705 const VMRegPair *regs,
706 AdapterFingerPrint* fingerprint) {
707 address i2c_entry = __ pc();
708
709 gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
710
711 address c2i_unverified_entry = __ pc();
712 Label skip_fixup;
713
714 Label ok;
715
716 Register holder = rscratch2;
717 Register receiver = j_rarg0;
718 Register tmp = r10; // A call-clobbered register not used for arg passing
719
720 // -------------------------------------------------------------------------
721 // Generate a C2I adapter. On entry we know rmethod holds the Method* during calls
722 // to the interpreter. The args start out packed in the compiled layout. They
723 // need to be unpacked into the interpreter layout. This will almost always
724 // require some stack space. We grow the current (compiled) stack, then repack
725 // the args. We finally end in a jump to the generic interpreter entry point.
726 // On exit from the interpreter, the interpreter will restore our SP (lest the
727 // compiled code, which relys solely on SP and not FP, get sick).
728
729 {
730 __ block_comment("c2i_unverified_entry {");
731 __ load_klass(rscratch1, receiver);
732 __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
733 __ cmp(rscratch1, tmp);
734 __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
735 __ br(Assembler::EQ, ok);
736 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
737
738 __ bind(ok);
739 // Method might have been compiled since the call site was patched to
740 // interpreted; if that is the case treat it as a miss so we can get
741 // the call site corrected.
742 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
743 __ cbz(rscratch1, skip_fixup);
744 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
745 __ block_comment("} c2i_unverified_entry");
746 }
747
748 address c2i_entry = __ pc();
749
750 // Class initialization barrier for static methods
751 address c2i_no_clinit_check_entry = NULL;
752 if (VM_Version::supports_fast_class_init_checks()) {
753 Label L_skip_barrier;
754
755 { // Bypass the barrier for non-static methods
756 __ ldrw(rscratch1, Address(rmethod, Method::access_flags_offset()));
757 __ andsw(zr, rscratch1, JVM_ACC_STATIC);
758 __ br(Assembler::EQ, L_skip_barrier); // non-static
759 }
760
761 __ load_method_holder(rscratch2, rmethod);
762 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
763 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
764
765 __ bind(L_skip_barrier);
766 c2i_no_clinit_check_entry = __ pc();
767 }
768
769 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
770 bs->c2i_entry_barrier(masm);
771
772 gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
773
774 __ flush();
775 return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
776 }
777
778 static int c_calling_convention_priv(const BasicType *sig_bt,
779 VMRegPair *regs,
780 VMRegPair *regs2,
781 int total_args_passed) {
782 assert(regs2 == NULL, "not needed on AArch64");
783
784 // We return the amount of VMRegImpl stack slots we need to reserve for all
785 // the arguments NOT counting out_preserve_stack_slots.
786
787 static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
788 c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5, c_rarg6, c_rarg7
789 };
790 static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
791 c_farg0, c_farg1, c_farg2, c_farg3,
792 c_farg4, c_farg5, c_farg6, c_farg7
793 };
794
795 uint int_args = 0;
803 case T_BYTE:
804 case T_SHORT:
805 case T_INT:
806 if (int_args < Argument::n_int_register_parameters_c) {
807 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
808 } else {
809 #ifdef __APPLE__
810 // Less-than word types are stored one after another.
811 // The code is unable to handle this so bailout.
812 return -1;
813 #endif
814 regs[i].set1(VMRegImpl::stack2reg(stk_args));
815 stk_args += 2;
816 }
817 break;
818 case T_LONG:
819 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
820 // fall through
821 case T_OBJECT:
822 case T_ARRAY:
823 case T_ADDRESS:
824 case T_METADATA:
825 if (int_args < Argument::n_int_register_parameters_c) {
826 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
827 } else {
828 regs[i].set2(VMRegImpl::stack2reg(stk_args));
829 stk_args += 2;
830 }
831 break;
832 case T_FLOAT:
833 if (fp_args < Argument::n_float_register_parameters_c) {
834 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
835 } else {
836 #ifdef __APPLE__
837 // Less-than word types are stored one after another.
838 // The code is unable to handle this so bailout.
839 return -1;
840 #endif
841 regs[i].set1(VMRegImpl::stack2reg(stk_args));
842 stk_args += 2;
1643 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1644 } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1645 freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1646 }
1647 #endif /* ASSERT */
1648 switch (in_sig_bt[i]) {
1649 case T_ARRAY:
1650 if (is_critical_native) {
1651 unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
1652 c_arg++;
1653 #ifdef ASSERT
1654 if (out_regs[c_arg].first()->is_Register()) {
1655 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1656 } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1657 freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1658 }
1659 #endif
1660 int_args++;
1661 break;
1662 }
1663 case T_OBJECT:
1664 assert(!is_critical_native, "no oop arguments");
1665 object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
1666 ((i == 0) && (!is_static)),
1667 &receiver_offset);
1668 int_args++;
1669 break;
1670 case T_VOID:
1671 break;
1672
1673 case T_FLOAT:
1674 float_move(masm, in_regs[i], out_regs[c_arg]);
1675 float_args++;
1676 break;
1677
1678 case T_DOUBLE:
1679 assert( i + 1 < total_in_args &&
1680 in_sig_bt[i + 1] == T_VOID &&
1681 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
1682 double_move(masm, in_regs[i], out_regs[c_arg]);
1759 Label lock_done;
1760
1761 if (method->is_synchronized()) {
1762 assert(!is_critical_native, "unhandled");
1763
1764 const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
1765
1766 // Get the handle (the 2nd argument)
1767 __ mov(oop_handle_reg, c_rarg1);
1768
1769 // Get address of the box
1770
1771 __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1772
1773 // Load the oop from the handle
1774 __ ldr(obj_reg, Address(oop_handle_reg, 0));
1775
1776 // Load (object->mark() | 1) into swap_reg %r0
1777 __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1778 __ orr(swap_reg, rscratch1, 1);
1779
1780 // Save (object->mark() | 1) into BasicLock's displaced header
1781 __ str(swap_reg, Address(lock_reg, mark_word_offset));
1782
1783 // src -> dest iff dest == r0 else r0 <- dest
1784 { Label here;
1785 __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, lock_done, /*fallthrough*/NULL);
1786 }
1787
1788 // Hmm should this move to the slow path code area???
1789
1790 // Test if the oopMark is an obvious stack pointer, i.e.,
1791 // 1) (mark & 3) == 0, and
1792 // 2) sp <= mark < mark + os::pagesize()
1793 // These 3 tests can be done by evaluating the following
1794 // expression: ((mark - sp) & (3 - os::vm_page_size())),
1795 // assuming both stack pointer and pagesize have their
1796 // least significant 2 bits clear.
1797 // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
1798
1824
1825 rt_call(masm, native_func);
1826
1827 __ bind(native_return);
1828
1829 intptr_t return_pc = (intptr_t) __ pc();
1830 oop_maps->add_gc_map(return_pc - start, map);
1831
1832 // Unpack native results.
1833 switch (ret_type) {
1834 case T_BOOLEAN: __ c2bool(r0); break;
1835 case T_CHAR : __ ubfx(r0, r0, 0, 16); break;
1836 case T_BYTE : __ sbfx(r0, r0, 0, 8); break;
1837 case T_SHORT : __ sbfx(r0, r0, 0, 16); break;
1838 case T_INT : __ sbfx(r0, r0, 0, 32); break;
1839 case T_DOUBLE :
1840 case T_FLOAT :
1841 // Result is in v0 we'll save as needed
1842 break;
1843 case T_ARRAY: // Really a handle
1844 case T_OBJECT: // Really a handle
1845 break; // can't de-handlize until after safepoint check
1846 case T_VOID: break;
1847 case T_LONG: break;
1848 default : ShouldNotReachHere();
1849 }
1850
1851 Label safepoint_in_progress, safepoint_in_progress_done;
1852 Label after_transition;
1853
1854 // If this is a critical native, check for a safepoint or suspend request after the call.
1855 // If a safepoint is needed, transition to native, then to native_trans to handle
1856 // safepoints like the native methods that are not critical natives.
1857 if (is_critical_native) {
1858 Label needs_safepoint;
1859 __ safepoint_poll(needs_safepoint, false /* at_return */, true /* acquire */, false /* in_nmethod */);
1860 __ ldrw(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
1861 __ cbnzw(rscratch1, needs_safepoint);
1862 __ b(after_transition);
1863 __ bind(needs_safepoint);
3066 #ifdef ASSERT
3067 __ str(zr, Address(rthread, JavaThread::exception_handler_pc_offset()));
3068 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
3069 #endif
3070 // Clear the exception oop so GC no longer processes it as a root.
3071 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
3072
3073 // r0: exception oop
3074 // r8: exception handler
3075 // r4: exception pc
3076 // Jump to handler
3077
3078 __ br(r8);
3079
3080 // Make sure all code is generated
3081 masm->flush();
3082
3083 // Set exception blob
3084 _exception_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
3085 }
3086
3087 // ---------------------------------------------------------------
3088
3089 class NativeInvokerGenerator : public StubCodeGenerator {
3090 address _call_target;
3091 int _shadow_space_bytes;
3092
3093 const GrowableArray<VMReg>& _input_registers;
3094 const GrowableArray<VMReg>& _output_registers;
3095
3096 int _frame_complete;
3097 int _framesize;
3098 OopMapSet* _oop_maps;
3099 public:
3100 NativeInvokerGenerator(CodeBuffer* buffer,
3101 address call_target,
3102 int shadow_space_bytes,
3103 const GrowableArray<VMReg>& input_registers,
3104 const GrowableArray<VMReg>& output_registers)
3105 : StubCodeGenerator(buffer, PrintMethodHandleStubs),
3318
3319 //////////////////////////////////////////////////////////////////////////////
3320
3321 __ block_comment("{ L_reguard");
3322 __ bind(L_reguard);
3323
3324 spill_output_registers();
3325
3326 rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
3327
3328 fill_output_registers();
3329
3330 __ b(L_after_reguard);
3331
3332 __ block_comment("} L_reguard");
3333
3334 //////////////////////////////////////////////////////////////////////////////
3335
3336 __ flush();
3337 }
3338 #endif // COMPILER2
|
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "classfile/symbolTable.hpp"
31 #include "code/codeCache.hpp"
32 #include "code/debugInfoRec.hpp"
33 #include "code/icBuffer.hpp"
34 #include "code/vtableStubs.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/shared/barrierSetAssembler.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "interpreter/interp_masm.hpp"
39 #include "logging/log.hpp"
40 #include "memory/resourceArea.hpp"
41 #include "nativeInst_aarch64.hpp"
42 #include "oops/compiledICHolder.hpp"
43 #include "oops/klass.inline.hpp"
44 #include "prims/methodHandles.hpp"
45 #include "runtime/jniHandles.hpp"
46 #include "runtime/safepointMechanism.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "runtime/signature.hpp"
49 #include "runtime/stubRoutines.hpp"
50 #include "runtime/vframeArray.hpp"
304 case T_SHORT:
305 case T_INT:
306 if (int_args < Argument::n_int_register_parameters_j) {
307 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
308 } else {
309 regs[i].set1(VMRegImpl::stack2reg(stk_args));
310 stk_args += 2;
311 }
312 break;
313 case T_VOID:
314 // halves of T_LONG or T_DOUBLE
315 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
316 regs[i].set_bad();
317 break;
318 case T_LONG:
319 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
320 // fall through
321 case T_OBJECT:
322 case T_ARRAY:
323 case T_ADDRESS:
324 case T_INLINE_TYPE:
325 if (int_args < Argument::n_int_register_parameters_j) {
326 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
327 } else {
328 regs[i].set2(VMRegImpl::stack2reg(stk_args));
329 stk_args += 2;
330 }
331 break;
332 case T_FLOAT:
333 if (fp_args < Argument::n_float_register_parameters_j) {
334 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
335 } else {
336 regs[i].set1(VMRegImpl::stack2reg(stk_args));
337 stk_args += 2;
338 }
339 break;
340 case T_DOUBLE:
341 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
342 if (fp_args < Argument::n_float_register_parameters_j) {
343 regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
344 } else {
345 regs[i].set2(VMRegImpl::stack2reg(stk_args));
346 stk_args += 2;
347 }
348 break;
349 default:
350 ShouldNotReachHere();
351 break;
352 }
353 }
354
355 return align_up(stk_args, 2);
356 }
357
358
359 const uint SharedRuntime::java_return_convention_max_int = Argument::n_int_register_parameters_j;
360 const uint SharedRuntime::java_return_convention_max_float = Argument::n_float_register_parameters_j;
361
362 int SharedRuntime::java_return_convention(const BasicType *sig_bt, VMRegPair *regs, int total_args_passed) {
363
364 // Create the mapping between argument positions and registers.
365
366 static const Register INT_ArgReg[java_return_convention_max_int] = {
367 r0 /* j_rarg7 */, j_rarg6, j_rarg5, j_rarg4, j_rarg3, j_rarg2, j_rarg1, j_rarg0
368 };
369
370 static const FloatRegister FP_ArgReg[java_return_convention_max_float] = {
371 j_farg0, j_farg1, j_farg2, j_farg3, j_farg4, j_farg5, j_farg6, j_farg7
372 };
373
374 uint int_args = 0;
375 uint fp_args = 0;
376
377 for (int i = 0; i < total_args_passed; i++) {
378 switch (sig_bt[i]) {
379 case T_BOOLEAN:
380 case T_CHAR:
381 case T_BYTE:
382 case T_SHORT:
383 case T_INT:
384 if (int_args < SharedRuntime::java_return_convention_max_int) {
385 regs[i].set1(INT_ArgReg[int_args]->as_VMReg());
386 int_args ++;
387 } else {
388 return -1;
389 }
390 break;
391 case T_VOID:
392 // halves of T_LONG or T_DOUBLE
393 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
394 regs[i].set_bad();
395 break;
396 case T_LONG:
397 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
398 // fall through
399 case T_OBJECT:
400 case T_ARRAY:
401 case T_ADDRESS:
402 // Should T_METADATA be added to java_calling_convention as well ?
403 case T_METADATA:
404 case T_INLINE_TYPE:
405 if (int_args < SharedRuntime::java_return_convention_max_int) {
406 regs[i].set2(INT_ArgReg[int_args]->as_VMReg());
407 int_args ++;
408 } else {
409 return -1;
410 }
411 break;
412 case T_FLOAT:
413 if (fp_args < SharedRuntime::java_return_convention_max_float) {
414 regs[i].set1(FP_ArgReg[fp_args]->as_VMReg());
415 fp_args ++;
416 } else {
417 return -1;
418 }
419 break;
420 case T_DOUBLE:
421 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
422 if (fp_args < SharedRuntime::java_return_convention_max_float) {
423 regs[i].set2(FP_ArgReg[fp_args]->as_VMReg());
424 fp_args ++;
425 } else {
426 return -1;
427 }
428 break;
429 default:
430 ShouldNotReachHere();
431 break;
432 }
433 }
434
435 return int_args + fp_args;
436 }
437
438 // Patch the callers callsite with entry to compiled code if it exists.
439 static void patch_callers_callsite(MacroAssembler *masm) {
440 Label L;
441 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
442 __ cbz(rscratch1, L);
443
444 __ enter();
445 __ push_CPU_state();
446
447 // VM needs caller's callsite
448 // VM needs target method
449 // This needs to be a long call since we will relocate this adapter to
450 // the codeBuffer and it may not reach
451
452 #ifndef PRODUCT
453 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
454 #endif
455
456 __ mov(c_rarg0, rmethod);
457 __ mov(c_rarg1, lr);
458 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
459 __ blr(rscratch1);
460
461 // Explicit isb required because fixup_callers_callsite may change the code
462 // stream.
463 __ safepoint_isb();
464
465 __ pop_CPU_state();
466 // restore sp
467 __ leave();
468 __ bind(L);
469 }
470
471 // For each inline type argument, sig includes the list of fields of
472 // the inline type. This utility function computes the number of
473 // arguments for the call if inline types are passed by reference (the
474 // calling convention the interpreter expects).
475 static int compute_total_args_passed_int(const GrowableArray<SigEntry>* sig_extended) {
476 int total_args_passed = 0;
477 if (InlineTypePassFieldsAsArgs) {
478 for (int i = 0; i < sig_extended->length(); i++) {
479 BasicType bt = sig_extended->at(i)._bt;
480 if (bt == T_INLINE_TYPE) {
481 // In sig_extended, an inline type argument starts with:
482 // T_INLINE_TYPE, followed by the types of the fields of the
483 // inline type and T_VOID to mark the end of the value
484 // type. Inline types are flattened so, for instance, in the
485 // case of an inline type with an int field and an inline type
486 // field that itself has 2 fields, an int and a long:
487 // T_INLINE_TYPE T_INT T_INLINE_TYPE T_INT T_LONG T_VOID (second
488 // slot for the T_LONG) T_VOID (inner T_INLINE_TYPE) T_VOID
489 // (outer T_INLINE_TYPE)
490 total_args_passed++;
491 int vt = 1;
492 do {
493 i++;
494 BasicType bt = sig_extended->at(i)._bt;
495 BasicType prev_bt = sig_extended->at(i-1)._bt;
496 if (bt == T_INLINE_TYPE) {
497 vt++;
498 } else if (bt == T_VOID &&
499 prev_bt != T_LONG &&
500 prev_bt != T_DOUBLE) {
501 vt--;
502 }
503 } while (vt != 0);
504 } else {
505 total_args_passed++;
506 }
507 }
508 } else {
509 total_args_passed = sig_extended->length();
510 }
511
512 return total_args_passed;
513 }
514
515
516 static void gen_c2i_adapter_helper(MacroAssembler* masm,
517 BasicType bt,
518 BasicType prev_bt,
519 size_t size_in_bytes,
520 const VMRegPair& reg_pair,
521 const Address& to,
522 Register tmp1,
523 Register tmp2,
524 Register tmp3,
525 int extraspace,
526 bool is_oop) {
527 assert(bt != T_INLINE_TYPE || !InlineTypePassFieldsAsArgs, "no inline type here");
528 if (bt == T_VOID) {
529 assert(prev_bt == T_LONG || prev_bt == T_DOUBLE, "missing half");
530 return;
531 }
532
533 // Say 4 args:
534 // i st_off
535 // 0 32 T_LONG
536 // 1 24 T_VOID
537 // 2 16 T_OBJECT
538 // 3 8 T_BOOL
539 // - 0 return address
540 //
541 // However to make thing extra confusing. Because we can fit a Java long/double in
542 // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
543 // leaves one slot empty and only stores to a single slot. In this case the
544 // slot that is occupied is the T_VOID slot. See I said it was confusing.
545
546 bool wide = (size_in_bytes == wordSize);
547 VMReg r_1 = reg_pair.first();
548 VMReg r_2 = reg_pair.second();
549 assert(r_2->is_valid() == wide, "invalid size");
550 if (!r_1->is_valid()) {
551 assert(!r_2->is_valid(), "");
552 return;
553 }
554
555 if (!r_1->is_FloatRegister()) {
556 Register val = tmp3;
557 if (r_1->is_stack()) {
558 // memory to memory use tmp3 (scratch registers are used by store_heap_oop)
559 int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
560 __ load_sized_value(val, Address(sp, ld_off), size_in_bytes, /* is_signed */ false);
561 } else {
562 val = r_1->as_Register();
563 }
564 assert_different_registers(to.base(), val, rscratch2, tmp1, tmp2);
565 if (is_oop) {
566 __ store_heap_oop(to, val, rscratch2, tmp1, tmp2, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
567 } else {
568 __ store_sized_value(to, val, size_in_bytes);
569 }
570 } else {
571 if (wide) {
572 __ strd(r_1->as_FloatRegister(), to);
573 } else {
574 // only a float use just part of the slot
575 __ strs(r_1->as_FloatRegister(), to);
576 }
577 }
578 }
579
580 static void gen_c2i_adapter(MacroAssembler *masm,
581 const GrowableArray<SigEntry>* sig_extended,
582 const VMRegPair *regs,
583 Label& skip_fixup,
584 address start,
585 OopMapSet* oop_maps,
586 int& frame_complete,
587 int& frame_size_in_words,
588 bool alloc_inline_receiver) {
589
590 // Before we get into the guts of the C2I adapter, see if we should be here
591 // at all. We've come from compiled code and are attempting to jump to the
592 // interpreter, which means the caller made a static call to get here
593 // (vcalls always get a compiled target if there is one). Check for a
594 // compiled target. If there is one, we need to patch the caller's call.
595 patch_callers_callsite(masm);
596
597 __ bind(skip_fixup);
598
599 // Name some registers to be used in the following code. We can use
600 // anything except r0-r7 which are arguments in the Java calling
601 // convention, rmethod (r12), and r13 which holds the outgoing sender
602 // SP for the interpreter.
603 Register buf_array = r10; // Array of buffered inline types
604 Register buf_oop = r11; // Buffered inline type oop
605 Register tmp1 = r15;
606 Register tmp2 = r16;
607 Register tmp3 = r17;
608
609 if (InlineTypePassFieldsAsArgs) {
610 // Is there an inline type argument?
611 bool has_inline_argument = false;
612 for (int i = 0; i < sig_extended->length() && !has_inline_argument; i++) {
613 has_inline_argument = (sig_extended->at(i)._bt == T_INLINE_TYPE);
614 }
615 if (has_inline_argument) {
616 // There is at least an inline type argument: we're coming from
617 // compiled code so we have no buffers to back the inline types
618 // Allocate the buffers here with a runtime call.
619 RegisterSaver reg_save(false /* save_vectors */);
620 OopMap* map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
621
622 frame_complete = __ offset();
623 address the_pc = __ pc();
624
625 Label retaddr;
626 __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
627
628 __ mov(c_rarg0, rthread);
629 __ mov(c_rarg1, rmethod);
630 __ mov(c_rarg2, (int64_t)alloc_inline_receiver);
631
632 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::allocate_inline_types)));
633 __ blr(rscratch1);
634 __ bind(retaddr);
635
636 oop_maps->add_gc_map(__ pc() - start, map);
637 __ reset_last_Java_frame(false);
638
639 reg_save.restore_live_registers(masm);
640
641 Label no_exception;
642 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
643 __ cbz(rscratch1, no_exception);
644
645 __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
646 __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
647 __ b(RuntimeAddress(StubRoutines::forward_exception_entry()));
648
649 __ bind(no_exception);
650
651 // We get an array of objects from the runtime call
652 __ get_vm_result(buf_array, rthread);
653 __ get_vm_result_2(rmethod, rthread); // TODO: required to keep the callee Method live?
654 }
655 }
656
657 // Since all args are passed on the stack, total_args_passed *
658 // Interpreter::stackElementSize is the space we need.
659
660 int total_args_passed = compute_total_args_passed_int(sig_extended);
661 int extraspace = total_args_passed * Interpreter::stackElementSize;
662
663 // stack is aligned, keep it that way
664 extraspace = align_up(extraspace, StackAlignmentInBytes);
665
666 // set senderSP value
667 __ mov(r13, sp);
668
669 __ sub(sp, sp, extraspace);
670
671 // Now write the args into the outgoing interpreter space
672
673 // next_arg_comp is the next argument from the compiler point of
674 // view (inline type fields are passed in registers/on the stack). In
675 // sig_extended, an inline type argument starts with: T_INLINE_TYPE,
676 // followed by the types of the fields of the inline type and T_VOID
677 // to mark the end of the inline type. ignored counts the number of
678 // T_INLINE_TYPE/T_VOID. next_vt_arg is the next inline type argument:
679 // used to get the buffer for that argument from the pool of buffers
680 // we allocated above and want to pass to the
681 // interpreter. next_arg_int is the next argument from the
682 // interpreter point of view (inline types are passed by reference).
683 for (int next_arg_comp = 0, ignored = 0, next_vt_arg = 0, next_arg_int = 0;
684 next_arg_comp < sig_extended->length(); next_arg_comp++) {
685 assert(ignored <= next_arg_comp, "shouldn't skip over more slots than there are arguments");
686 assert(next_arg_int <= total_args_passed, "more arguments for the interpreter than expected?");
687 BasicType bt = sig_extended->at(next_arg_comp)._bt;
688 int st_off = (total_args_passed - next_arg_int - 1) * Interpreter::stackElementSize;
689 if (!InlineTypePassFieldsAsArgs || bt != T_INLINE_TYPE) {
690 int next_off = st_off - Interpreter::stackElementSize;
691 const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : st_off;
692 const VMRegPair reg_pair = regs[next_arg_comp-ignored];
693 size_t size_in_bytes = reg_pair.second()->is_valid() ? 8 : 4;
694 gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
695 size_in_bytes, reg_pair, Address(sp, offset), tmp1, tmp2, tmp3, extraspace, false);
696 next_arg_int++;
697 #ifdef ASSERT
698 if (bt == T_LONG || bt == T_DOUBLE) {
699 // Overwrite the unused slot with known junk
700 __ mov(rscratch1, CONST64(0xdeadffffdeadaaaa));
701 __ str(rscratch1, Address(sp, st_off));
702 }
703 #endif /* ASSERT */
704 } else {
705 ignored++;
706 // get the buffer from the just allocated pool of buffers
707 int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + next_vt_arg * type2aelembytes(T_INLINE_TYPE);
708 __ load_heap_oop(buf_oop, Address(buf_array, index));
709 next_vt_arg++; next_arg_int++;
710 int vt = 1;
711 // write fields we get from compiled code in registers/stack
712 // slots to the buffer: we know we are done with that inline type
713 // argument when we hit the T_VOID that acts as an end of inline
714 // type delimiter for this inline type. Inline types are flattened
715 // so we might encounter embedded inline types. Each entry in
716 // sig_extended contains a field offset in the buffer.
717 do {
718 next_arg_comp++;
719 BasicType bt = sig_extended->at(next_arg_comp)._bt;
720 BasicType prev_bt = sig_extended->at(next_arg_comp - 1)._bt;
721 if (bt == T_INLINE_TYPE) {
722 vt++;
723 ignored++;
724 } else if (bt == T_VOID && prev_bt != T_LONG && prev_bt != T_DOUBLE) {
725 vt--;
726 ignored++;
727 } else {
728 int off = sig_extended->at(next_arg_comp)._offset;
729 assert(off > 0, "offset in object should be positive");
730 size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
731 bool is_oop = is_reference_type(bt);
732 gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
733 size_in_bytes, regs[next_arg_comp-ignored], Address(buf_oop, off), tmp1, tmp2, tmp3, extraspace, is_oop);
734 }
735 } while (vt != 0);
736 // pass the buffer to the interpreter
737 __ str(buf_oop, Address(sp, st_off));
738 }
739 }
740
741 __ mov(esp, sp); // Interp expects args on caller's expression stack
742
743 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
744 __ br(rscratch1);
745 }
746
747 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int comp_args_on_stack, const GrowableArray<SigEntry>* sig, const VMRegPair *regs) {
748
749
750 // Note: r13 contains the senderSP on entry. We must preserve it since
751 // we may do a i2c -> c2i transition if we lose a race where compiled
752 // code goes non-entrant while we get args ready.
753
754 // In addition we use r13 to locate all the interpreter args because
755 // we must align the stack to 16 bytes.
756
757 // Adapters are frameless.
758
759 // An i2c adapter is frameless because the *caller* frame, which is
760 // interpreted, routinely repairs its own esp (from
761 // interpreter_frame_last_sp), even if a callee has modified the
762 // stack pointer. It also recalculates and aligns sp.
763
764 // A c2i adapter is frameless because the *callee* frame, which is
765 // interpreted, routinely repairs its caller's sp (from sender_sp,
766 // which is set up via the senderSP register).
767
768 // In other words, if *either* the caller or callee is interpreted, we can
788 range_check(masm, rax, r11,
789 Interpreter::code()->code_start(), Interpreter::code()->code_end(),
790 L_ok);
791 if (StubRoutines::code1() != NULL)
792 range_check(masm, rax, r11,
793 StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
794 L_ok);
795 if (StubRoutines::code2() != NULL)
796 range_check(masm, rax, r11,
797 StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
798 L_ok);
799 const char* msg = "i2c adapter must return to an interpreter frame";
800 __ block_comment(msg);
801 __ stop(msg);
802 __ bind(L_ok);
803 __ block_comment("} verify_i2ce ");
804 #endif
805 }
806
807 // Cut-out for having no stack args.
808 int comp_words_on_stack = 0;
809 if (comp_args_on_stack) {
810 comp_words_on_stack = align_up(comp_args_on_stack * VMRegImpl::stack_slot_size, wordSize) >> LogBytesPerWord;
811 __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
812 __ andr(sp, rscratch1, -16);
813 }
814
815 // Will jump to the compiled code just as if compiled code was doing it.
816 // Pre-load the register-jump target early, to schedule it better.
817 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_inline_offset())));
818
819 #if INCLUDE_JVMCI
820 if (EnableJVMCI) {
821 // check if this call should be routed towards a specific entry point
822 __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
823 Label no_alternative_target;
824 __ cbz(rscratch2, no_alternative_target);
825 __ mov(rscratch1, rscratch2);
826 __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
827 __ bind(no_alternative_target);
828 }
829 #endif // INCLUDE_JVMCI
830
831 int total_args_passed = sig->length();
832
833 // Now generate the shuffle code.
834 for (int i = 0; i < total_args_passed; i++) {
835 BasicType bt = sig->at(i)._bt;
836
837 assert(bt != T_INLINE_TYPE, "i2c adapter doesn't unpack inline typ args");
838 if (bt == T_VOID) {
839 assert(i > 0 && (sig->at(i - 1)._bt == T_LONG || sig->at(i - 1)._bt == T_DOUBLE), "missing half");
840 continue;
841 }
842
843 // Pick up 0, 1 or 2 words from SP+offset.
844 assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(), "scrambled load targets?");
845
846 // Load in argument order going down.
847 int ld_off = (total_args_passed - i - 1) * Interpreter::stackElementSize;
848 // Point to interpreter value (vs. tag)
849 int next_off = ld_off - Interpreter::stackElementSize;
850 //
851 //
852 //
853 VMReg r_1 = regs[i].first();
854 VMReg r_2 = regs[i].second();
855 if (!r_1->is_valid()) {
856 assert(!r_2->is_valid(), "");
857 continue;
858 }
859 if (r_1->is_stack()) {
860 // Convert stack slot to an SP offset (+ wordSize to account for return address )
861 int st_off = regs[i].first()->reg2stack() * VMRegImpl::stack_slot_size;
862 if (!r_2->is_valid()) {
863 // sign extend???
864 __ ldrsw(rscratch2, Address(esp, ld_off));
865 __ str(rscratch2, Address(sp, st_off));
866 } else {
867 //
868 // We are using two optoregs. This can be either T_OBJECT,
869 // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
870 // two slots but only uses one for thr T_LONG or T_DOUBLE case
871 // So we must adjust where to pick up the data to match the
872 // interpreter.
873 //
874 // Interpreter local[n] == MSW, local[n+1] == LSW however locals
875 // are accessed as negative so LSW is at LOW address
876
877 // ld_off is MSW so get LSW
878 const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;
879 __ ldr(rscratch2, Address(esp, offset));
880 // st_off is LSW (i.e. reg.first())
881 __ str(rscratch2, Address(sp, st_off));
882 }
883 } else if (r_1->is_Register()) { // Register argument
884 Register r = r_1->as_Register();
885 if (r_2->is_valid()) {
886 //
887 // We are using two VMRegs. This can be either T_OBJECT,
888 // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
889 // two slots but only uses one for thr T_LONG or T_DOUBLE case
890 // So we must adjust where to pick up the data to match the
891 // interpreter.
892
893 const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;
894
895 // this can be a misaligned move
896 __ ldr(r, Address(esp, offset));
897 } else {
898 // sign extend and use a full word?
899 __ ldrw(r, Address(esp, ld_off));
900 }
901 } else {
902 if (!r_2->is_valid()) {
903 __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
904 } else {
905 __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
906 }
907 }
908 }
909
910
911 // 6243940 We might end up in handle_wrong_method if
912 // the callee is deoptimized as we race thru here. If that
913 // happens we don't want to take a safepoint because the
914 // caller frame will look interpreted and arguments are now
915 // "compiled" so it is much better to make this transition
916 // invisible to the stack walking code. Unfortunately if
917 // we try and find the callee by normal means a safepoint
918 // is possible. So we stash the desired callee in the thread
919 // and the vm will find there should this case occur.
920
921 __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));
922 __ br(rscratch1);
923 }
924
925 static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) {
926
927 Label ok;
928
929 Register holder = rscratch2;
930 Register receiver = j_rarg0;
931 Register tmp = r10; // A call-clobbered register not used for arg passing
932
933 // -------------------------------------------------------------------------
934 // Generate a C2I adapter. On entry we know rmethod holds the Method* during calls
935 // to the interpreter. The args start out packed in the compiled layout. They
936 // need to be unpacked into the interpreter layout. This will almost always
937 // require some stack space. We grow the current (compiled) stack, then repack
938 // the args. We finally end in a jump to the generic interpreter entry point.
939 // On exit from the interpreter, the interpreter will restore our SP (lest the
940 // compiled code, which relys solely on SP and not FP, get sick).
941
942 {
943 __ block_comment("c2i_unverified_entry {");
944 __ load_klass(rscratch1, receiver);
945 __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
946 __ cmp(rscratch1, tmp);
947 __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
948 __ br(Assembler::EQ, ok);
949 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
950
951 __ bind(ok);
952 // Method might have been compiled since the call site was patched to
953 // interpreted; if that is the case treat it as a miss so we can get
954 // the call site corrected.
955 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
956 __ cbz(rscratch1, skip_fixup);
957 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
958 __ block_comment("} c2i_unverified_entry");
959 }
960 }
961
962
963 // ---------------------------------------------------------------
964 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,
965 int comp_args_on_stack,
966 const GrowableArray<SigEntry>* sig,
967 const VMRegPair* regs,
968 const GrowableArray<SigEntry>* sig_cc,
969 const VMRegPair* regs_cc,
970 const GrowableArray<SigEntry>* sig_cc_ro,
971 const VMRegPair* regs_cc_ro,
972 AdapterFingerPrint* fingerprint,
973 AdapterBlob*& new_adapter,
974 bool allocate_code_blob) {
975
976 address i2c_entry = __ pc();
977 gen_i2c_adapter(masm, comp_args_on_stack, sig, regs);
978
979 address c2i_unverified_entry = __ pc();
980 Label skip_fixup;
981
982 gen_inline_cache_check(masm, skip_fixup);
983
984 OopMapSet* oop_maps = new OopMapSet();
985 int frame_complete = CodeOffsets::frame_never_safe;
986 int frame_size_in_words = 0;
987
988 // Scalarized c2i adapter with non-scalarized receiver (i.e., don't pack receiver)
989 address c2i_inline_ro_entry = __ pc();
990 if (regs_cc != regs_cc_ro) {
991 gen_c2i_adapter(masm, sig_cc_ro, regs_cc_ro, skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, false);
992 skip_fixup.reset();
993 }
994
995 // Scalarized c2i adapter
996 address c2i_entry = __ pc();
997
998 // Class initialization barrier for static methods
999 address c2i_no_clinit_check_entry = NULL;
1000 if (VM_Version::supports_fast_class_init_checks()) {
1001 Label L_skip_barrier;
1002
1003 { // Bypass the barrier for non-static methods
1004 __ ldrw(rscratch1, Address(rmethod, Method::access_flags_offset()));
1005 __ andsw(zr, rscratch1, JVM_ACC_STATIC);
1006 __ br(Assembler::EQ, L_skip_barrier); // non-static
1007 }
1008
1009 __ load_method_holder(rscratch2, rmethod);
1010 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1011 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1012
1013 __ bind(L_skip_barrier);
1014 c2i_no_clinit_check_entry = __ pc();
1015 }
1016
1017 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1018 bs->c2i_entry_barrier(masm);
1019
1020 gen_c2i_adapter(masm, sig_cc, regs_cc, skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, true);
1021
1022 address c2i_unverified_inline_entry = c2i_unverified_entry;
1023
1024 // Non-scalarized c2i adapter
1025 address c2i_inline_entry = c2i_entry;
1026 if (regs != regs_cc) {
1027 Label inline_entry_skip_fixup;
1028 c2i_unverified_inline_entry = __ pc();
1029 gen_inline_cache_check(masm, inline_entry_skip_fixup);
1030
1031 c2i_inline_entry = __ pc();
1032 gen_c2i_adapter(masm, sig, regs, inline_entry_skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, false);
1033 }
1034
1035 __ flush();
1036
1037 // The c2i adapter might safepoint and trigger a GC. The caller must make sure that
1038 // the GC knows about the location of oop argument locations passed to the c2i adapter.
1039 if (allocate_code_blob) {
1040 bool caller_must_gc_arguments = (regs != regs_cc);
1041 new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
1042 }
1043
1044 return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry, c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
1045 }
1046
1047 static int c_calling_convention_priv(const BasicType *sig_bt,
1048 VMRegPair *regs,
1049 VMRegPair *regs2,
1050 int total_args_passed) {
1051 assert(regs2 == NULL, "not needed on AArch64");
1052
1053 // We return the amount of VMRegImpl stack slots we need to reserve for all
1054 // the arguments NOT counting out_preserve_stack_slots.
1055
1056 static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1057 c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5, c_rarg6, c_rarg7
1058 };
1059 static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
1060 c_farg0, c_farg1, c_farg2, c_farg3,
1061 c_farg4, c_farg5, c_farg6, c_farg7
1062 };
1063
1064 uint int_args = 0;
1072 case T_BYTE:
1073 case T_SHORT:
1074 case T_INT:
1075 if (int_args < Argument::n_int_register_parameters_c) {
1076 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
1077 } else {
1078 #ifdef __APPLE__
1079 // Less-than word types are stored one after another.
1080 // The code is unable to handle this so bailout.
1081 return -1;
1082 #endif
1083 regs[i].set1(VMRegImpl::stack2reg(stk_args));
1084 stk_args += 2;
1085 }
1086 break;
1087 case T_LONG:
1088 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
1089 // fall through
1090 case T_OBJECT:
1091 case T_ARRAY:
1092 case T_INLINE_TYPE:
1093 case T_ADDRESS:
1094 case T_METADATA:
1095 if (int_args < Argument::n_int_register_parameters_c) {
1096 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
1097 } else {
1098 regs[i].set2(VMRegImpl::stack2reg(stk_args));
1099 stk_args += 2;
1100 }
1101 break;
1102 case T_FLOAT:
1103 if (fp_args < Argument::n_float_register_parameters_c) {
1104 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
1105 } else {
1106 #ifdef __APPLE__
1107 // Less-than word types are stored one after another.
1108 // The code is unable to handle this so bailout.
1109 return -1;
1110 #endif
1111 regs[i].set1(VMRegImpl::stack2reg(stk_args));
1112 stk_args += 2;
1913 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1914 } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1915 freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1916 }
1917 #endif /* ASSERT */
1918 switch (in_sig_bt[i]) {
1919 case T_ARRAY:
1920 if (is_critical_native) {
1921 unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
1922 c_arg++;
1923 #ifdef ASSERT
1924 if (out_regs[c_arg].first()->is_Register()) {
1925 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1926 } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1927 freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1928 }
1929 #endif
1930 int_args++;
1931 break;
1932 }
1933 case T_INLINE_TYPE:
1934 case T_OBJECT:
1935 assert(!is_critical_native, "no oop arguments");
1936 object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
1937 ((i == 0) && (!is_static)),
1938 &receiver_offset);
1939 int_args++;
1940 break;
1941 case T_VOID:
1942 break;
1943
1944 case T_FLOAT:
1945 float_move(masm, in_regs[i], out_regs[c_arg]);
1946 float_args++;
1947 break;
1948
1949 case T_DOUBLE:
1950 assert( i + 1 < total_in_args &&
1951 in_sig_bt[i + 1] == T_VOID &&
1952 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
1953 double_move(masm, in_regs[i], out_regs[c_arg]);
2030 Label lock_done;
2031
2032 if (method->is_synchronized()) {
2033 assert(!is_critical_native, "unhandled");
2034
2035 const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
2036
2037 // Get the handle (the 2nd argument)
2038 __ mov(oop_handle_reg, c_rarg1);
2039
2040 // Get address of the box
2041
2042 __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
2043
2044 // Load the oop from the handle
2045 __ ldr(obj_reg, Address(oop_handle_reg, 0));
2046
2047 // Load (object->mark() | 1) into swap_reg %r0
2048 __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2049 __ orr(swap_reg, rscratch1, 1);
2050 if (EnableValhalla) {
2051 // Mask inline_type bit such that we go to the slow path if object is an inline type
2052 __ andr(swap_reg, swap_reg, ~((int) markWord::inline_type_bit_in_place));
2053 }
2054
2055 // Save (object->mark() | 1) into BasicLock's displaced header
2056 __ str(swap_reg, Address(lock_reg, mark_word_offset));
2057
2058 // src -> dest iff dest == r0 else r0 <- dest
2059 { Label here;
2060 __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, lock_done, /*fallthrough*/NULL);
2061 }
2062
2063 // Hmm should this move to the slow path code area???
2064
2065 // Test if the oopMark is an obvious stack pointer, i.e.,
2066 // 1) (mark & 3) == 0, and
2067 // 2) sp <= mark < mark + os::pagesize()
2068 // These 3 tests can be done by evaluating the following
2069 // expression: ((mark - sp) & (3 - os::vm_page_size())),
2070 // assuming both stack pointer and pagesize have their
2071 // least significant 2 bits clear.
2072 // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
2073
2099
2100 rt_call(masm, native_func);
2101
2102 __ bind(native_return);
2103
2104 intptr_t return_pc = (intptr_t) __ pc();
2105 oop_maps->add_gc_map(return_pc - start, map);
2106
2107 // Unpack native results.
2108 switch (ret_type) {
2109 case T_BOOLEAN: __ c2bool(r0); break;
2110 case T_CHAR : __ ubfx(r0, r0, 0, 16); break;
2111 case T_BYTE : __ sbfx(r0, r0, 0, 8); break;
2112 case T_SHORT : __ sbfx(r0, r0, 0, 16); break;
2113 case T_INT : __ sbfx(r0, r0, 0, 32); break;
2114 case T_DOUBLE :
2115 case T_FLOAT :
2116 // Result is in v0 we'll save as needed
2117 break;
2118 case T_ARRAY: // Really a handle
2119 case T_INLINE_TYPE: // Really a handle
2120 case T_OBJECT: // Really a handle
2121 break; // can't de-handlize until after safepoint check
2122 case T_VOID: break;
2123 case T_LONG: break;
2124 default : ShouldNotReachHere();
2125 }
2126
2127 Label safepoint_in_progress, safepoint_in_progress_done;
2128 Label after_transition;
2129
2130 // If this is a critical native, check for a safepoint or suspend request after the call.
2131 // If a safepoint is needed, transition to native, then to native_trans to handle
2132 // safepoints like the native methods that are not critical natives.
2133 if (is_critical_native) {
2134 Label needs_safepoint;
2135 __ safepoint_poll(needs_safepoint, false /* at_return */, true /* acquire */, false /* in_nmethod */);
2136 __ ldrw(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
2137 __ cbnzw(rscratch1, needs_safepoint);
2138 __ b(after_transition);
2139 __ bind(needs_safepoint);
3342 #ifdef ASSERT
3343 __ str(zr, Address(rthread, JavaThread::exception_handler_pc_offset()));
3344 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
3345 #endif
3346 // Clear the exception oop so GC no longer processes it as a root.
3347 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
3348
3349 // r0: exception oop
3350 // r8: exception handler
3351 // r4: exception pc
3352 // Jump to handler
3353
3354 __ br(r8);
3355
3356 // Make sure all code is generated
3357 masm->flush();
3358
3359 // Set exception blob
3360 _exception_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
3361 }
3362 #endif // COMPILER2
3363
3364 BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) {
3365 BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K);
3366 CodeBuffer buffer(buf);
3367 short buffer_locs[20];
3368 buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
3369 sizeof(buffer_locs)/sizeof(relocInfo));
3370
3371 MacroAssembler _masm(&buffer);
3372 MacroAssembler* masm = &_masm;
3373
3374 const Array<SigEntry>* sig_vk = vk->extended_sig();
3375 const Array<VMRegPair>* regs = vk->return_regs();
3376
3377 int pack_fields_jobject_off = __ offset();
3378 // Resolve pre-allocated buffer from JNI handle.
3379 // We cannot do this in generate_call_stub() because it requires GC code to be initialized.
3380 Register Rresult = r14; // See StubGenerator::generate_call_stub().
3381 __ ldr(r0, Address(Rresult));
3382 __ resolve_jobject(r0 /* value */,
3383 rthread /* thread */,
3384 r12 /* tmp */);
3385 __ str(r0, Address(Rresult));
3386
3387 int pack_fields_off = __ offset();
3388
3389 int j = 1;
3390 for (int i = 0; i < sig_vk->length(); i++) {
3391 BasicType bt = sig_vk->at(i)._bt;
3392 if (bt == T_INLINE_TYPE) {
3393 continue;
3394 }
3395 if (bt == T_VOID) {
3396 if (sig_vk->at(i-1)._bt == T_LONG ||
3397 sig_vk->at(i-1)._bt == T_DOUBLE) {
3398 j++;
3399 }
3400 continue;
3401 }
3402 int off = sig_vk->at(i)._offset;
3403 VMRegPair pair = regs->at(j);
3404 VMReg r_1 = pair.first();
3405 VMReg r_2 = pair.second();
3406 Address to(r0, off);
3407 if (bt == T_FLOAT) {
3408 __ strs(r_1->as_FloatRegister(), to);
3409 } else if (bt == T_DOUBLE) {
3410 __ strd(r_1->as_FloatRegister(), to);
3411 } else if (bt == T_OBJECT || bt == T_ARRAY) {
3412 Register val = r_1->as_Register();
3413 assert_different_registers(r0, val);
3414 // We don't need barriers because the destination is a newly allocated object.
3415 // Also, we cannot use store_heap_oop(to, val) because it uses r8 as tmp.
3416 if (UseCompressedOops) {
3417 __ encode_heap_oop(val);
3418 __ str(val, to);
3419 } else {
3420 __ str(val, to);
3421 }
3422 } else {
3423 assert(is_java_primitive(bt), "unexpected basic type");
3424 assert_different_registers(r0, r_1->as_Register());
3425 size_t size_in_bytes = type2aelembytes(bt);
3426 __ store_sized_value(to, r_1->as_Register(), size_in_bytes);
3427 }
3428 j++;
3429 }
3430 assert(j == regs->length(), "missed a field?");
3431
3432 __ ret(lr);
3433
3434 int unpack_fields_off = __ offset();
3435
3436 j = 1;
3437 for (int i = 0; i < sig_vk->length(); i++) {
3438 BasicType bt = sig_vk->at(i)._bt;
3439 if (bt == T_INLINE_TYPE) {
3440 continue;
3441 }
3442 if (bt == T_VOID) {
3443 if (sig_vk->at(i-1)._bt == T_LONG ||
3444 sig_vk->at(i-1)._bt == T_DOUBLE) {
3445 j++;
3446 }
3447 continue;
3448 }
3449 int off = sig_vk->at(i)._offset;
3450 assert(off > 0, "offset in object should be positive");
3451 VMRegPair pair = regs->at(j);
3452 VMReg r_1 = pair.first();
3453 VMReg r_2 = pair.second();
3454 Address from(r0, off);
3455 if (bt == T_FLOAT) {
3456 __ ldrs(r_1->as_FloatRegister(), from);
3457 } else if (bt == T_DOUBLE) {
3458 __ ldrd(r_1->as_FloatRegister(), from);
3459 } else if (bt == T_OBJECT || bt == T_ARRAY) {
3460 assert_different_registers(r0, r_1->as_Register());
3461 __ load_heap_oop(r_1->as_Register(), from);
3462 } else {
3463 assert(is_java_primitive(bt), "unexpected basic type");
3464 assert_different_registers(r0, r_1->as_Register());
3465
3466 size_t size_in_bytes = type2aelembytes(bt);
3467 __ load_sized_value(r_1->as_Register(), from, size_in_bytes, bt != T_CHAR && bt != T_BOOLEAN);
3468 }
3469 j++;
3470 }
3471 assert(j == regs->length(), "missed a field?");
3472
3473 if (StressInlineTypeReturnedAsFields) {
3474 __ load_klass(r0, r0);
3475 __ orr(r0, r0, 1);
3476 }
3477
3478 __ ret(lr);
3479
3480 __ flush();
3481
3482 return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
3483 }
3484
3485 // ---------------------------------------------------------------
3486
3487 class NativeInvokerGenerator : public StubCodeGenerator {
3488 address _call_target;
3489 int _shadow_space_bytes;
3490
3491 const GrowableArray<VMReg>& _input_registers;
3492 const GrowableArray<VMReg>& _output_registers;
3493
3494 int _frame_complete;
3495 int _framesize;
3496 OopMapSet* _oop_maps;
3497 public:
3498 NativeInvokerGenerator(CodeBuffer* buffer,
3499 address call_target,
3500 int shadow_space_bytes,
3501 const GrowableArray<VMReg>& input_registers,
3502 const GrowableArray<VMReg>& output_registers)
3503 : StubCodeGenerator(buffer, PrintMethodHandleStubs),
3716
3717 //////////////////////////////////////////////////////////////////////////////
3718
3719 __ block_comment("{ L_reguard");
3720 __ bind(L_reguard);
3721
3722 spill_output_registers();
3723
3724 rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
3725
3726 fill_output_registers();
3727
3728 __ b(L_after_reguard);
3729
3730 __ block_comment("} L_reguard");
3731
3732 //////////////////////////////////////////////////////////////////////////////
3733
3734 __ flush();
3735 }
|