10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "code/codeCache.hpp"
31 #include "code/debugInfoRec.hpp"
32 #include "code/icBuffer.hpp"
33 #include "code/vtableStubs.hpp"
34 #include "compiler/oopMap.hpp"
35 #include "gc/shared/barrierSetAssembler.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "interpreter/interp_masm.hpp"
38 #include "logging/log.hpp"
39 #include "memory/resourceArea.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/compiledICHolder.hpp"
42 #include "oops/klass.inline.hpp"
43 #include "prims/methodHandles.hpp"
44 #include "runtime/jniHandles.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/signature.hpp"
48 #include "runtime/stubRoutines.hpp"
49 #include "runtime/vframeArray.hpp"
340 case T_SHORT:
341 case T_INT:
342 if (int_args < Argument::n_int_register_parameters_j) {
343 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
344 } else {
345 regs[i].set1(VMRegImpl::stack2reg(stk_args));
346 stk_args += 2;
347 }
348 break;
349 case T_VOID:
350 // halves of T_LONG or T_DOUBLE
351 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
352 regs[i].set_bad();
353 break;
354 case T_LONG:
355 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
356 // fall through
357 case T_OBJECT:
358 case T_ARRAY:
359 case T_ADDRESS:
360 if (int_args < Argument::n_int_register_parameters_j) {
361 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
362 } else {
363 regs[i].set2(VMRegImpl::stack2reg(stk_args));
364 stk_args += 2;
365 }
366 break;
367 case T_FLOAT:
368 if (fp_args < Argument::n_float_register_parameters_j) {
369 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
370 } else {
371 regs[i].set1(VMRegImpl::stack2reg(stk_args));
372 stk_args += 2;
373 }
374 break;
375 case T_DOUBLE:
376 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
377 if (fp_args < Argument::n_float_register_parameters_j) {
378 regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
379 } else {
380 regs[i].set2(VMRegImpl::stack2reg(stk_args));
381 stk_args += 2;
382 }
383 break;
384 default:
385 ShouldNotReachHere();
386 break;
387 }
388 }
389
390 return align_up(stk_args, 2);
391 }
392
393 // Patch the callers callsite with entry to compiled code if it exists.
394 static void patch_callers_callsite(MacroAssembler *masm) {
395 Label L;
396 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
397 __ cbz(rscratch1, L);
398
399 __ enter();
400 __ push_CPU_state();
401
402 // VM needs caller's callsite
403 // VM needs target method
404 // This needs to be a long call since we will relocate this adapter to
405 // the codeBuffer and it may not reach
406
407 #ifndef PRODUCT
408 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
409 #endif
410
411 __ mov(c_rarg0, rmethod);
412 __ mov(c_rarg1, lr);
413 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
414 __ blr(rscratch1);
415
416 // Explicit isb required because fixup_callers_callsite may change the code
417 // stream.
418 __ safepoint_isb();
419
420 __ pop_CPU_state();
421 // restore sp
422 __ leave();
423 __ bind(L);
424 }
425
426 static void gen_c2i_adapter(MacroAssembler *masm,
427 int total_args_passed,
428 int comp_args_on_stack,
429 const BasicType *sig_bt,
430 const VMRegPair *regs,
431 Label& skip_fixup) {
432 // Before we get into the guts of the C2I adapter, see if we should be here
433 // at all. We've come from compiled code and are attempting to jump to the
434 // interpreter, which means the caller made a static call to get here
435 // (vcalls always get a compiled target if there is one). Check for a
436 // compiled target. If there is one, we need to patch the caller's call.
437 patch_callers_callsite(masm);
438
439 __ bind(skip_fixup);
440
441 int words_pushed = 0;
442
443 // Since all args are passed on the stack, total_args_passed *
444 // Interpreter::stackElementSize is the space we need.
445
446 int extraspace = total_args_passed * Interpreter::stackElementSize;
447
448 __ mov(r13, sp);
449
450 // stack is aligned, keep it that way
451 extraspace = align_up(extraspace, 2*wordSize);
452
453 if (extraspace)
454 __ sub(sp, sp, extraspace);
455
456 // Now write the args into the outgoing interpreter space
457 for (int i = 0; i < total_args_passed; i++) {
458 if (sig_bt[i] == T_VOID) {
459 assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
460 continue;
461 }
462
463 // offset to start parameters
464 int st_off = (total_args_passed - i - 1) * Interpreter::stackElementSize;
465 int next_off = st_off - Interpreter::stackElementSize;
466
467 // Say 4 args:
468 // i st_off
469 // 0 32 T_LONG
470 // 1 24 T_VOID
471 // 2 16 T_OBJECT
472 // 3 8 T_BOOL
473 // - 0 return address
474 //
475 // However to make thing extra confusing. Because we can fit a Java long/double in
476 // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
477 // leaves one slot empty and only stores to a single slot. In this case the
478 // slot that is occupied is the T_VOID slot. See I said it was confusing.
479
480 VMReg r_1 = regs[i].first();
481 VMReg r_2 = regs[i].second();
482 if (!r_1->is_valid()) {
483 assert(!r_2->is_valid(), "");
484 continue;
485 }
486 if (r_1->is_stack()) {
487 // memory to memory use rscratch1
488 int ld_off = (r_1->reg2stack() * VMRegImpl::stack_slot_size
489 + extraspace
490 + words_pushed * wordSize);
491 if (!r_2->is_valid()) {
492 // sign extend??
493 __ ldrw(rscratch1, Address(sp, ld_off));
494 __ str(rscratch1, Address(sp, st_off));
495
496 } else {
497
498 __ ldr(rscratch1, Address(sp, ld_off));
499
500 // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
501 // T_DOUBLE and T_LONG use two slots in the interpreter
502 if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
503 // ld_off == LSW, ld_off+wordSize == MSW
504 // st_off == MSW, next_off == LSW
505 __ str(rscratch1, Address(sp, next_off));
506 #ifdef ASSERT
507 // Overwrite the unused slot with known junk
508 __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaaaull);
509 __ str(rscratch1, Address(sp, st_off));
510 #endif /* ASSERT */
511 } else {
512 __ str(rscratch1, Address(sp, st_off));
513 }
514 }
515 } else if (r_1->is_Register()) {
516 Register r = r_1->as_Register();
517 if (!r_2->is_valid()) {
518 // must be only an int (or less ) so move only 32bits to slot
519 // why not sign extend??
520 __ str(r, Address(sp, st_off));
521 } else {
522 // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
523 // T_DOUBLE and T_LONG use two slots in the interpreter
524 if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
525 // jlong/double in gpr
526 #ifdef ASSERT
527 // Overwrite the unused slot with known junk
528 __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaabull);
529 __ str(rscratch1, Address(sp, st_off));
530 #endif /* ASSERT */
531 __ str(r, Address(sp, next_off));
532 } else {
533 __ str(r, Address(sp, st_off));
534 }
535 }
536 } else {
537 assert(r_1->is_FloatRegister(), "");
538 if (!r_2->is_valid()) {
539 // only a float use just part of the slot
540 __ strs(r_1->as_FloatRegister(), Address(sp, st_off));
541 } else {
542 #ifdef ASSERT
543 // Overwrite the unused slot with known junk
544 __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaacull);
545 __ str(rscratch1, Address(sp, st_off));
546 #endif /* ASSERT */
547 __ strd(r_1->as_FloatRegister(), Address(sp, next_off));
548 }
549 }
550 }
551
552 __ mov(esp, sp); // Interp expects args on caller's expression stack
553
554 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
555 __ br(rscratch1);
556 }
557
558
559 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
560 int total_args_passed,
561 int comp_args_on_stack,
562 const BasicType *sig_bt,
563 const VMRegPair *regs) {
564
565 // Note: r13 contains the senderSP on entry. We must preserve it since
566 // we may do a i2c -> c2i transition if we lose a race where compiled
567 // code goes non-entrant while we get args ready.
568
569 // In addition we use r13 to locate all the interpreter args because
570 // we must align the stack to 16 bytes.
571
572 // Adapters are frameless.
573
574 // An i2c adapter is frameless because the *caller* frame, which is
575 // interpreted, routinely repairs its own esp (from
576 // interpreter_frame_last_sp), even if a callee has modified the
577 // stack pointer. It also recalculates and aligns sp.
578
579 // A c2i adapter is frameless because the *callee* frame, which is
580 // interpreted, routinely repairs its caller's sp (from sender_sp,
581 // which is set up via the senderSP register).
582
583 // In other words, if *either* the caller or callee is interpreted, we can
603 range_check(masm, rax, r11,
604 Interpreter::code()->code_start(), Interpreter::code()->code_end(),
605 L_ok);
606 if (StubRoutines::code1() != NULL)
607 range_check(masm, rax, r11,
608 StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
609 L_ok);
610 if (StubRoutines::code2() != NULL)
611 range_check(masm, rax, r11,
612 StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
613 L_ok);
614 const char* msg = "i2c adapter must return to an interpreter frame";
615 __ block_comment(msg);
616 __ stop(msg);
617 __ bind(L_ok);
618 __ block_comment("} verify_i2ce ");
619 #endif
620 }
621
622 // Cut-out for having no stack args.
623 int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
624 if (comp_args_on_stack) {
625 __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
626 __ andr(sp, rscratch1, -16);
627 }
628
629 // Will jump to the compiled code just as if compiled code was doing it.
630 // Pre-load the register-jump target early, to schedule it better.
631 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_offset())));
632
633 #if INCLUDE_JVMCI
634 if (EnableJVMCI) {
635 // check if this call should be routed towards a specific entry point
636 __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
637 Label no_alternative_target;
638 __ cbz(rscratch2, no_alternative_target);
639 __ mov(rscratch1, rscratch2);
640 __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
641 __ bind(no_alternative_target);
642 }
643 #endif // INCLUDE_JVMCI
644
645 // Now generate the shuffle code.
646 for (int i = 0; i < total_args_passed; i++) {
647 if (sig_bt[i] == T_VOID) {
648 assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
649 continue;
650 }
651
652 // Pick up 0, 1 or 2 words from SP+offset.
653
654 assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
655 "scrambled load targets?");
656 // Load in argument order going down.
657 int ld_off = (total_args_passed - i - 1)*Interpreter::stackElementSize;
658 // Point to interpreter value (vs. tag)
659 int next_off = ld_off - Interpreter::stackElementSize;
660 //
661 //
662 //
663 VMReg r_1 = regs[i].first();
664 VMReg r_2 = regs[i].second();
665 if (!r_1->is_valid()) {
666 assert(!r_2->is_valid(), "");
667 continue;
668 }
669 if (r_1->is_stack()) {
670 // Convert stack slot to an SP offset (+ wordSize to account for return address )
671 int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size;
672 if (!r_2->is_valid()) {
673 // sign extend???
674 __ ldrsw(rscratch2, Address(esp, ld_off));
675 __ str(rscratch2, Address(sp, st_off));
676 } else {
677 //
678 // We are using two optoregs. This can be either T_OBJECT,
679 // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
680 // two slots but only uses one for thr T_LONG or T_DOUBLE case
681 // So we must adjust where to pick up the data to match the
682 // interpreter.
683 //
684 // Interpreter local[n] == MSW, local[n+1] == LSW however locals
685 // are accessed as negative so LSW is at LOW address
686
687 // ld_off is MSW so get LSW
688 const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
689 next_off : ld_off;
690 __ ldr(rscratch2, Address(esp, offset));
691 // st_off is LSW (i.e. reg.first())
692 __ str(rscratch2, Address(sp, st_off));
693 }
694 } else if (r_1->is_Register()) { // Register argument
695 Register r = r_1->as_Register();
696 if (r_2->is_valid()) {
697 //
698 // We are using two VMRegs. This can be either T_OBJECT,
699 // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
700 // two slots but only uses one for thr T_LONG or T_DOUBLE case
701 // So we must adjust where to pick up the data to match the
702 // interpreter.
703
704 const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
705 next_off : ld_off;
706
707 // this can be a misaligned move
708 __ ldr(r, Address(esp, offset));
709 } else {
710 // sign extend and use a full word?
711 __ ldrw(r, Address(esp, ld_off));
712 }
713 } else {
714 if (!r_2->is_valid()) {
715 __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
716 } else {
717 __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
718 }
719 }
720 }
721
722 // 6243940 We might end up in handle_wrong_method if
723 // the callee is deoptimized as we race thru here. If that
724 // happens we don't want to take a safepoint because the
725 // caller frame will look interpreted and arguments are now
726 // "compiled" so it is much better to make this transition
727 // invisible to the stack walking code. Unfortunately if
728 // we try and find the callee by normal means a safepoint
729 // is possible. So we stash the desired callee in the thread
730 // and the vm will find there should this case occur.
731
732 __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));
733
734 __ br(rscratch1);
735 }
736
737 // ---------------------------------------------------------------
738 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
739 int total_args_passed,
740 int comp_args_on_stack,
741 const BasicType *sig_bt,
742 const VMRegPair *regs,
743 AdapterFingerPrint* fingerprint) {
744 address i2c_entry = __ pc();
745
746 gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
747
748 address c2i_unverified_entry = __ pc();
749 Label skip_fixup;
750
751 Label ok;
752
753 Register holder = rscratch2;
754 Register receiver = j_rarg0;
755 Register tmp = r10; // A call-clobbered register not used for arg passing
756
757 // -------------------------------------------------------------------------
758 // Generate a C2I adapter. On entry we know rmethod holds the Method* during calls
759 // to the interpreter. The args start out packed in the compiled layout. They
760 // need to be unpacked into the interpreter layout. This will almost always
761 // require some stack space. We grow the current (compiled) stack, then repack
762 // the args. We finally end in a jump to the generic interpreter entry point.
763 // On exit from the interpreter, the interpreter will restore our SP (lest the
764 // compiled code, which relys solely on SP and not FP, get sick).
765
766 {
767 __ block_comment("c2i_unverified_entry {");
768 __ load_klass(rscratch1, receiver);
769 __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
770 __ cmp(rscratch1, tmp);
771 __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
772 __ br(Assembler::EQ, ok);
773 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
774
775 __ bind(ok);
776 // Method might have been compiled since the call site was patched to
777 // interpreted; if that is the case treat it as a miss so we can get
778 // the call site corrected.
779 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
780 __ cbz(rscratch1, skip_fixup);
781 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
782 __ block_comment("} c2i_unverified_entry");
783 }
784
785 address c2i_entry = __ pc();
786
787 // Class initialization barrier for static methods
788 address c2i_no_clinit_check_entry = NULL;
789 if (VM_Version::supports_fast_class_init_checks()) {
790 Label L_skip_barrier;
791
792 { // Bypass the barrier for non-static methods
793 __ ldrw(rscratch1, Address(rmethod, Method::access_flags_offset()));
794 __ andsw(zr, rscratch1, JVM_ACC_STATIC);
795 __ br(Assembler::EQ, L_skip_barrier); // non-static
796 }
797
798 __ load_method_holder(rscratch2, rmethod);
799 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
800 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
801
802 __ bind(L_skip_barrier);
803 c2i_no_clinit_check_entry = __ pc();
804 }
805
806 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
807 bs->c2i_entry_barrier(masm);
808
809 gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
810
811 __ flush();
812 return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
813 }
814
815 static int c_calling_convention_priv(const BasicType *sig_bt,
816 VMRegPair *regs,
817 VMRegPair *regs2,
818 int total_args_passed) {
819 assert(regs2 == NULL, "not needed on AArch64");
820
821 // We return the amount of VMRegImpl stack slots we need to reserve for all
822 // the arguments NOT counting out_preserve_stack_slots.
823
824 static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
825 c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5, c_rarg6, c_rarg7
826 };
827 static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
828 c_farg0, c_farg1, c_farg2, c_farg3,
829 c_farg4, c_farg5, c_farg6, c_farg7
830 };
831
832 uint int_args = 0;
840 case T_BYTE:
841 case T_SHORT:
842 case T_INT:
843 if (int_args < Argument::n_int_register_parameters_c) {
844 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
845 } else {
846 #ifdef __APPLE__
847 // Less-than word types are stored one after another.
848 // The code is unable to handle this so bailout.
849 return -1;
850 #endif
851 regs[i].set1(VMRegImpl::stack2reg(stk_args));
852 stk_args += 2;
853 }
854 break;
855 case T_LONG:
856 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
857 // fall through
858 case T_OBJECT:
859 case T_ARRAY:
860 case T_ADDRESS:
861 case T_METADATA:
862 if (int_args < Argument::n_int_register_parameters_c) {
863 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
864 } else {
865 regs[i].set2(VMRegImpl::stack2reg(stk_args));
866 stk_args += 2;
867 }
868 break;
869 case T_FLOAT:
870 if (fp_args < Argument::n_float_register_parameters_c) {
871 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
872 } else {
873 #ifdef __APPLE__
874 // Less-than word types are stored one after another.
875 // The code is unable to handle this so bailout.
876 return -1;
877 #endif
878 regs[i].set1(VMRegImpl::stack2reg(stk_args));
879 stk_args += 2;
1514 int temploc = -1;
1515 for (int ai = 0; ai < arg_order.length(); ai += 2) {
1516 int i = arg_order.at(ai);
1517 int c_arg = arg_order.at(ai + 1);
1518 __ block_comment(err_msg("move %d -> %d", i, c_arg));
1519 assert(c_arg != -1 && i != -1, "wrong order");
1520 #ifdef ASSERT
1521 if (in_regs[i].first()->is_Register()) {
1522 assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
1523 } else if (in_regs[i].first()->is_FloatRegister()) {
1524 assert(!freg_destroyed[in_regs[i].first()->as_FloatRegister()->encoding()], "destroyed reg!");
1525 }
1526 if (out_regs[c_arg].first()->is_Register()) {
1527 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1528 } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1529 freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1530 }
1531 #endif /* ASSERT */
1532 switch (in_sig_bt[i]) {
1533 case T_ARRAY:
1534 case T_OBJECT:
1535 object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
1536 ((i == 0) && (!is_static)),
1537 &receiver_offset);
1538 int_args++;
1539 break;
1540 case T_VOID:
1541 break;
1542
1543 case T_FLOAT:
1544 float_move(masm, in_regs[i], out_regs[c_arg]);
1545 float_args++;
1546 break;
1547
1548 case T_DOUBLE:
1549 assert( i + 1 < total_in_args &&
1550 in_sig_bt[i + 1] == T_VOID &&
1551 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
1552 double_move(masm, in_regs[i], out_regs[c_arg]);
1553 float_args++;
1628 Label slow_path_lock;
1629 Label lock_done;
1630
1631 if (method->is_synchronized()) {
1632
1633 const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
1634
1635 // Get the handle (the 2nd argument)
1636 __ mov(oop_handle_reg, c_rarg1);
1637
1638 // Get address of the box
1639
1640 __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1641
1642 // Load the oop from the handle
1643 __ ldr(obj_reg, Address(oop_handle_reg, 0));
1644
1645 // Load (object->mark() | 1) into swap_reg %r0
1646 __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1647 __ orr(swap_reg, rscratch1, 1);
1648
1649 // Save (object->mark() | 1) into BasicLock's displaced header
1650 __ str(swap_reg, Address(lock_reg, mark_word_offset));
1651
1652 // src -> dest iff dest == r0 else r0 <- dest
1653 { Label here;
1654 __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, lock_done, /*fallthrough*/NULL);
1655 }
1656
1657 // Hmm should this move to the slow path code area???
1658
1659 // Test if the oopMark is an obvious stack pointer, i.e.,
1660 // 1) (mark & 3) == 0, and
1661 // 2) sp <= mark < mark + os::pagesize()
1662 // These 3 tests can be done by evaluating the following
1663 // expression: ((mark - sp) & (3 - os::vm_page_size())),
1664 // assuming both stack pointer and pagesize have their
1665 // least significant 2 bits clear.
1666 // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
1667
1691
1692 rt_call(masm, native_func);
1693
1694 __ bind(native_return);
1695
1696 intptr_t return_pc = (intptr_t) __ pc();
1697 oop_maps->add_gc_map(return_pc - start, map);
1698
1699 // Unpack native results.
1700 switch (ret_type) {
1701 case T_BOOLEAN: __ c2bool(r0); break;
1702 case T_CHAR : __ ubfx(r0, r0, 0, 16); break;
1703 case T_BYTE : __ sbfx(r0, r0, 0, 8); break;
1704 case T_SHORT : __ sbfx(r0, r0, 0, 16); break;
1705 case T_INT : __ sbfx(r0, r0, 0, 32); break;
1706 case T_DOUBLE :
1707 case T_FLOAT :
1708 // Result is in v0 we'll save as needed
1709 break;
1710 case T_ARRAY: // Really a handle
1711 case T_OBJECT: // Really a handle
1712 break; // can't de-handlize until after safepoint check
1713 case T_VOID: break;
1714 case T_LONG: break;
1715 default : ShouldNotReachHere();
1716 }
1717
1718 Label safepoint_in_progress, safepoint_in_progress_done;
1719 Label after_transition;
1720
1721 // Switch thread to "native transition" state before reading the synchronization state.
1722 // This additional state is necessary because reading and testing the synchronization
1723 // state is not atomic w.r.t. GC, as this scenario demonstrates:
1724 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
1725 // VM thread changes sync state to synchronizing and suspends threads for GC.
1726 // Thread A is resumed to finish this native method, but doesn't block here since it
1727 // didn't see any synchronization is progress, and escapes.
1728 __ mov(rscratch1, _thread_in_native_trans);
1729
1730 __ strw(rscratch1, Address(rthread, JavaThread::thread_state_offset()));
2915 #ifdef ASSERT
2916 __ str(zr, Address(rthread, JavaThread::exception_handler_pc_offset()));
2917 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
2918 #endif
2919 // Clear the exception oop so GC no longer processes it as a root.
2920 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
2921
2922 // r0: exception oop
2923 // r8: exception handler
2924 // r4: exception pc
2925 // Jump to handler
2926
2927 __ br(r8);
2928
2929 // Make sure all code is generated
2930 masm->flush();
2931
2932 // Set exception blob
2933 _exception_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
2934 }
2935
2936 // ---------------------------------------------------------------
2937
2938 class NativeInvokerGenerator : public StubCodeGenerator {
2939 address _call_target;
2940 int _shadow_space_bytes;
2941
2942 const GrowableArray<VMReg>& _input_registers;
2943 const GrowableArray<VMReg>& _output_registers;
2944
2945 int _frame_complete;
2946 int _framesize;
2947 OopMapSet* _oop_maps;
2948 public:
2949 NativeInvokerGenerator(CodeBuffer* buffer,
2950 address call_target,
2951 int shadow_space_bytes,
2952 const GrowableArray<VMReg>& input_registers,
2953 const GrowableArray<VMReg>& output_registers)
2954 : StubCodeGenerator(buffer, PrintMethodHandleStubs),
3167
3168 //////////////////////////////////////////////////////////////////////////////
3169
3170 __ block_comment("{ L_reguard");
3171 __ bind(L_reguard);
3172
3173 spill_output_registers();
3174
3175 rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
3176
3177 fill_output_registers();
3178
3179 __ b(L_after_reguard);
3180
3181 __ block_comment("} L_reguard");
3182
3183 //////////////////////////////////////////////////////////////////////////////
3184
3185 __ flush();
3186 }
3187 #endif // COMPILER2
|
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "classfile/symbolTable.hpp"
31 #include "code/codeCache.hpp"
32 #include "code/debugInfoRec.hpp"
33 #include "code/icBuffer.hpp"
34 #include "code/vtableStubs.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/shared/barrierSetAssembler.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "interpreter/interp_masm.hpp"
39 #include "logging/log.hpp"
40 #include "memory/resourceArea.hpp"
41 #include "nativeInst_aarch64.hpp"
42 #include "oops/compiledICHolder.hpp"
43 #include "oops/klass.inline.hpp"
44 #include "prims/methodHandles.hpp"
45 #include "runtime/jniHandles.hpp"
46 #include "runtime/safepointMechanism.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "runtime/signature.hpp"
49 #include "runtime/stubRoutines.hpp"
50 #include "runtime/vframeArray.hpp"
341 case T_SHORT:
342 case T_INT:
343 if (int_args < Argument::n_int_register_parameters_j) {
344 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
345 } else {
346 regs[i].set1(VMRegImpl::stack2reg(stk_args));
347 stk_args += 2;
348 }
349 break;
350 case T_VOID:
351 // halves of T_LONG or T_DOUBLE
352 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
353 regs[i].set_bad();
354 break;
355 case T_LONG:
356 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
357 // fall through
358 case T_OBJECT:
359 case T_ARRAY:
360 case T_ADDRESS:
361 case T_INLINE_TYPE:
362 if (int_args < Argument::n_int_register_parameters_j) {
363 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
364 } else {
365 regs[i].set2(VMRegImpl::stack2reg(stk_args));
366 stk_args += 2;
367 }
368 break;
369 case T_FLOAT:
370 if (fp_args < Argument::n_float_register_parameters_j) {
371 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
372 } else {
373 regs[i].set1(VMRegImpl::stack2reg(stk_args));
374 stk_args += 2;
375 }
376 break;
377 case T_DOUBLE:
378 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
379 if (fp_args < Argument::n_float_register_parameters_j) {
380 regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
381 } else {
382 regs[i].set2(VMRegImpl::stack2reg(stk_args));
383 stk_args += 2;
384 }
385 break;
386 default:
387 ShouldNotReachHere();
388 break;
389 }
390 }
391
392 return align_up(stk_args, 2);
393 }
394
395
396 const uint SharedRuntime::java_return_convention_max_int = Argument::n_int_register_parameters_j;
397 const uint SharedRuntime::java_return_convention_max_float = Argument::n_float_register_parameters_j;
398
399 int SharedRuntime::java_return_convention(const BasicType *sig_bt, VMRegPair *regs, int total_args_passed) {
400
401 // Create the mapping between argument positions and registers.
402
403 static const Register INT_ArgReg[java_return_convention_max_int] = {
404 r0 /* j_rarg7 */, j_rarg6, j_rarg5, j_rarg4, j_rarg3, j_rarg2, j_rarg1, j_rarg0
405 };
406
407 static const FloatRegister FP_ArgReg[java_return_convention_max_float] = {
408 j_farg0, j_farg1, j_farg2, j_farg3, j_farg4, j_farg5, j_farg6, j_farg7
409 };
410
411 uint int_args = 0;
412 uint fp_args = 0;
413
414 for (int i = 0; i < total_args_passed; i++) {
415 switch (sig_bt[i]) {
416 case T_BOOLEAN:
417 case T_CHAR:
418 case T_BYTE:
419 case T_SHORT:
420 case T_INT:
421 if (int_args < SharedRuntime::java_return_convention_max_int) {
422 regs[i].set1(INT_ArgReg[int_args]->as_VMReg());
423 int_args ++;
424 } else {
425 return -1;
426 }
427 break;
428 case T_VOID:
429 // halves of T_LONG or T_DOUBLE
430 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
431 regs[i].set_bad();
432 break;
433 case T_LONG:
434 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
435 // fall through
436 case T_OBJECT:
437 case T_ARRAY:
438 case T_ADDRESS:
439 // Should T_METADATA be added to java_calling_convention as well ?
440 case T_METADATA:
441 case T_INLINE_TYPE:
442 if (int_args < SharedRuntime::java_return_convention_max_int) {
443 regs[i].set2(INT_ArgReg[int_args]->as_VMReg());
444 int_args ++;
445 } else {
446 return -1;
447 }
448 break;
449 case T_FLOAT:
450 if (fp_args < SharedRuntime::java_return_convention_max_float) {
451 regs[i].set1(FP_ArgReg[fp_args]->as_VMReg());
452 fp_args ++;
453 } else {
454 return -1;
455 }
456 break;
457 case T_DOUBLE:
458 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
459 if (fp_args < SharedRuntime::java_return_convention_max_float) {
460 regs[i].set2(FP_ArgReg[fp_args]->as_VMReg());
461 fp_args ++;
462 } else {
463 return -1;
464 }
465 break;
466 default:
467 ShouldNotReachHere();
468 break;
469 }
470 }
471
472 return int_args + fp_args;
473 }
474
475 // Patch the callers callsite with entry to compiled code if it exists.
476 static void patch_callers_callsite(MacroAssembler *masm) {
477 Label L;
478 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
479 __ cbz(rscratch1, L);
480
481 __ enter();
482 __ push_CPU_state();
483
484 // VM needs caller's callsite
485 // VM needs target method
486 // This needs to be a long call since we will relocate this adapter to
487 // the codeBuffer and it may not reach
488
489 #ifndef PRODUCT
490 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
491 #endif
492
493 __ mov(c_rarg0, rmethod);
494 __ mov(c_rarg1, lr);
495 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
496 __ blr(rscratch1);
497
498 // Explicit isb required because fixup_callers_callsite may change the code
499 // stream.
500 __ safepoint_isb();
501
502 __ pop_CPU_state();
503 // restore sp
504 __ leave();
505 __ bind(L);
506 }
507
508 // For each inline type argument, sig includes the list of fields of
509 // the inline type. This utility function computes the number of
510 // arguments for the call if inline types are passed by reference (the
511 // calling convention the interpreter expects).
512 static int compute_total_args_passed_int(const GrowableArray<SigEntry>* sig_extended) {
513 int total_args_passed = 0;
514 if (InlineTypePassFieldsAsArgs) {
515 for (int i = 0; i < sig_extended->length(); i++) {
516 BasicType bt = sig_extended->at(i)._bt;
517 if (bt == T_INLINE_TYPE) {
518 // In sig_extended, an inline type argument starts with:
519 // T_INLINE_TYPE, followed by the types of the fields of the
520 // inline type and T_VOID to mark the end of the value
521 // type. Inline types are flattened so, for instance, in the
522 // case of an inline type with an int field and an inline type
523 // field that itself has 2 fields, an int and a long:
524 // T_INLINE_TYPE T_INT T_INLINE_TYPE T_INT T_LONG T_VOID (second
525 // slot for the T_LONG) T_VOID (inner T_INLINE_TYPE) T_VOID
526 // (outer T_INLINE_TYPE)
527 total_args_passed++;
528 int vt = 1;
529 do {
530 i++;
531 BasicType bt = sig_extended->at(i)._bt;
532 BasicType prev_bt = sig_extended->at(i-1)._bt;
533 if (bt == T_INLINE_TYPE) {
534 vt++;
535 } else if (bt == T_VOID &&
536 prev_bt != T_LONG &&
537 prev_bt != T_DOUBLE) {
538 vt--;
539 }
540 } while (vt != 0);
541 } else {
542 total_args_passed++;
543 }
544 }
545 } else {
546 total_args_passed = sig_extended->length();
547 }
548
549 return total_args_passed;
550 }
551
552
553 static void gen_c2i_adapter_helper(MacroAssembler* masm,
554 BasicType bt,
555 BasicType prev_bt,
556 size_t size_in_bytes,
557 const VMRegPair& reg_pair,
558 const Address& to,
559 Register tmp1,
560 Register tmp2,
561 Register tmp3,
562 int extraspace,
563 bool is_oop) {
564 assert(bt != T_INLINE_TYPE || !InlineTypePassFieldsAsArgs, "no inline type here");
565 if (bt == T_VOID) {
566 assert(prev_bt == T_LONG || prev_bt == T_DOUBLE, "missing half");
567 return;
568 }
569
570 // Say 4 args:
571 // i st_off
572 // 0 32 T_LONG
573 // 1 24 T_VOID
574 // 2 16 T_OBJECT
575 // 3 8 T_BOOL
576 // - 0 return address
577 //
578 // However to make thing extra confusing. Because we can fit a Java long/double in
579 // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
580 // leaves one slot empty and only stores to a single slot. In this case the
581 // slot that is occupied is the T_VOID slot. See I said it was confusing.
582
583 bool wide = (size_in_bytes == wordSize);
584 VMReg r_1 = reg_pair.first();
585 VMReg r_2 = reg_pair.second();
586 assert(r_2->is_valid() == wide, "invalid size");
587 if (!r_1->is_valid()) {
588 assert(!r_2->is_valid(), "");
589 return;
590 }
591
592 if (!r_1->is_FloatRegister()) {
593 Register val = tmp3;
594 if (r_1->is_stack()) {
595 // memory to memory use tmp3 (scratch registers are used by store_heap_oop)
596 int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
597 __ load_sized_value(val, Address(sp, ld_off), size_in_bytes, /* is_signed */ false);
598 } else {
599 val = r_1->as_Register();
600 }
601 assert_different_registers(to.base(), val, rscratch2, tmp1, tmp2);
602 if (is_oop) {
603 __ store_heap_oop(to, val, rscratch2, tmp1, tmp2, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
604 } else {
605 __ store_sized_value(to, val, size_in_bytes);
606 }
607 } else {
608 if (wide) {
609 __ strd(r_1->as_FloatRegister(), to);
610 } else {
611 // only a float use just part of the slot
612 __ strs(r_1->as_FloatRegister(), to);
613 }
614 }
615 }
616
617 static void gen_c2i_adapter(MacroAssembler *masm,
618 const GrowableArray<SigEntry>* sig_extended,
619 const VMRegPair *regs,
620 Label& skip_fixup,
621 address start,
622 OopMapSet* oop_maps,
623 int& frame_complete,
624 int& frame_size_in_words,
625 bool alloc_inline_receiver) {
626
627 // Before we get into the guts of the C2I adapter, see if we should be here
628 // at all. We've come from compiled code and are attempting to jump to the
629 // interpreter, which means the caller made a static call to get here
630 // (vcalls always get a compiled target if there is one). Check for a
631 // compiled target. If there is one, we need to patch the caller's call.
632 patch_callers_callsite(masm);
633
634 __ bind(skip_fixup);
635
636 // Name some registers to be used in the following code. We can use
637 // anything except r0-r7 which are arguments in the Java calling
638 // convention, rmethod (r12), and r13 which holds the outgoing sender
639 // SP for the interpreter.
640 Register buf_array = r10; // Array of buffered inline types
641 Register buf_oop = r11; // Buffered inline type oop
642 Register tmp1 = r15;
643 Register tmp2 = r16;
644 Register tmp3 = r17;
645
646 if (InlineTypePassFieldsAsArgs) {
647 // Is there an inline type argument?
648 bool has_inline_argument = false;
649 for (int i = 0; i < sig_extended->length() && !has_inline_argument; i++) {
650 has_inline_argument = (sig_extended->at(i)._bt == T_INLINE_TYPE);
651 }
652 if (has_inline_argument) {
653 // There is at least an inline type argument: we're coming from
654 // compiled code so we have no buffers to back the inline types
655 // Allocate the buffers here with a runtime call.
656 RegisterSaver reg_save(false /* save_vectors */);
657 OopMap* map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
658
659 frame_complete = __ offset();
660 address the_pc = __ pc();
661
662 Label retaddr;
663 __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
664
665 __ mov(c_rarg0, rthread);
666 __ mov(c_rarg1, rmethod);
667 __ mov(c_rarg2, (int64_t)alloc_inline_receiver);
668
669 __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::allocate_inline_types)));
670 __ blr(rscratch1);
671 __ bind(retaddr);
672
673 oop_maps->add_gc_map(__ pc() - start, map);
674 __ reset_last_Java_frame(false);
675
676 reg_save.restore_live_registers(masm);
677
678 Label no_exception;
679 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
680 __ cbz(rscratch1, no_exception);
681
682 __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
683 __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
684 __ b(RuntimeAddress(StubRoutines::forward_exception_entry()));
685
686 __ bind(no_exception);
687
688 // We get an array of objects from the runtime call
689 __ get_vm_result(buf_array, rthread);
690 __ get_vm_result_2(rmethod, rthread); // TODO: required to keep the callee Method live?
691 }
692 }
693
694 // Since all args are passed on the stack, total_args_passed *
695 // Interpreter::stackElementSize is the space we need.
696
697 int total_args_passed = compute_total_args_passed_int(sig_extended);
698 int extraspace = total_args_passed * Interpreter::stackElementSize;
699
700 // stack is aligned, keep it that way
701 extraspace = align_up(extraspace, StackAlignmentInBytes);
702
703 // set senderSP value
704 __ mov(r13, sp);
705
706 __ sub(sp, sp, extraspace);
707
708 // Now write the args into the outgoing interpreter space
709
710 // next_arg_comp is the next argument from the compiler point of
711 // view (inline type fields are passed in registers/on the stack). In
712 // sig_extended, an inline type argument starts with: T_INLINE_TYPE,
713 // followed by the types of the fields of the inline type and T_VOID
714 // to mark the end of the inline type. ignored counts the number of
715 // T_INLINE_TYPE/T_VOID. next_vt_arg is the next inline type argument:
716 // used to get the buffer for that argument from the pool of buffers
717 // we allocated above and want to pass to the
718 // interpreter. next_arg_int is the next argument from the
719 // interpreter point of view (inline types are passed by reference).
720 for (int next_arg_comp = 0, ignored = 0, next_vt_arg = 0, next_arg_int = 0;
721 next_arg_comp < sig_extended->length(); next_arg_comp++) {
722 assert(ignored <= next_arg_comp, "shouldn't skip over more slots than there are arguments");
723 assert(next_arg_int <= total_args_passed, "more arguments for the interpreter than expected?");
724 BasicType bt = sig_extended->at(next_arg_comp)._bt;
725 int st_off = (total_args_passed - next_arg_int - 1) * Interpreter::stackElementSize;
726 if (!InlineTypePassFieldsAsArgs || bt != T_INLINE_TYPE) {
727 int next_off = st_off - Interpreter::stackElementSize;
728 const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : st_off;
729 const VMRegPair reg_pair = regs[next_arg_comp-ignored];
730 size_t size_in_bytes = reg_pair.second()->is_valid() ? 8 : 4;
731 gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
732 size_in_bytes, reg_pair, Address(sp, offset), tmp1, tmp2, tmp3, extraspace, false);
733 next_arg_int++;
734 #ifdef ASSERT
735 if (bt == T_LONG || bt == T_DOUBLE) {
736 // Overwrite the unused slot with known junk
737 __ mov(rscratch1, CONST64(0xdeadffffdeadaaaa));
738 __ str(rscratch1, Address(sp, st_off));
739 }
740 #endif /* ASSERT */
741 } else {
742 ignored++;
743 // get the buffer from the just allocated pool of buffers
744 int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + next_vt_arg * type2aelembytes(T_INLINE_TYPE);
745 __ load_heap_oop(buf_oop, Address(buf_array, index));
746 next_vt_arg++; next_arg_int++;
747 int vt = 1;
748 // write fields we get from compiled code in registers/stack
749 // slots to the buffer: we know we are done with that inline type
750 // argument when we hit the T_VOID that acts as an end of inline
751 // type delimiter for this inline type. Inline types are flattened
752 // so we might encounter embedded inline types. Each entry in
753 // sig_extended contains a field offset in the buffer.
754 do {
755 next_arg_comp++;
756 BasicType bt = sig_extended->at(next_arg_comp)._bt;
757 BasicType prev_bt = sig_extended->at(next_arg_comp - 1)._bt;
758 if (bt == T_INLINE_TYPE) {
759 vt++;
760 ignored++;
761 } else if (bt == T_VOID && prev_bt != T_LONG && prev_bt != T_DOUBLE) {
762 vt--;
763 ignored++;
764 } else {
765 int off = sig_extended->at(next_arg_comp)._offset;
766 assert(off > 0, "offset in object should be positive");
767 size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
768 bool is_oop = is_reference_type(bt);
769 gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
770 size_in_bytes, regs[next_arg_comp-ignored], Address(buf_oop, off), tmp1, tmp2, tmp3, extraspace, is_oop);
771 }
772 } while (vt != 0);
773 // pass the buffer to the interpreter
774 __ str(buf_oop, Address(sp, st_off));
775 }
776 }
777
778 __ mov(esp, sp); // Interp expects args on caller's expression stack
779
780 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
781 __ br(rscratch1);
782 }
783
784 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int comp_args_on_stack, const GrowableArray<SigEntry>* sig, const VMRegPair *regs) {
785
786
787 // Note: r13 contains the senderSP on entry. We must preserve it since
788 // we may do a i2c -> c2i transition if we lose a race where compiled
789 // code goes non-entrant while we get args ready.
790
791 // In addition we use r13 to locate all the interpreter args because
792 // we must align the stack to 16 bytes.
793
794 // Adapters are frameless.
795
796 // An i2c adapter is frameless because the *caller* frame, which is
797 // interpreted, routinely repairs its own esp (from
798 // interpreter_frame_last_sp), even if a callee has modified the
799 // stack pointer. It also recalculates and aligns sp.
800
801 // A c2i adapter is frameless because the *callee* frame, which is
802 // interpreted, routinely repairs its caller's sp (from sender_sp,
803 // which is set up via the senderSP register).
804
805 // In other words, if *either* the caller or callee is interpreted, we can
825 range_check(masm, rax, r11,
826 Interpreter::code()->code_start(), Interpreter::code()->code_end(),
827 L_ok);
828 if (StubRoutines::code1() != NULL)
829 range_check(masm, rax, r11,
830 StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
831 L_ok);
832 if (StubRoutines::code2() != NULL)
833 range_check(masm, rax, r11,
834 StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
835 L_ok);
836 const char* msg = "i2c adapter must return to an interpreter frame";
837 __ block_comment(msg);
838 __ stop(msg);
839 __ bind(L_ok);
840 __ block_comment("} verify_i2ce ");
841 #endif
842 }
843
844 // Cut-out for having no stack args.
845 int comp_words_on_stack = 0;
846 if (comp_args_on_stack) {
847 comp_words_on_stack = align_up(comp_args_on_stack * VMRegImpl::stack_slot_size, wordSize) >> LogBytesPerWord;
848 __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
849 __ andr(sp, rscratch1, -16);
850 }
851
852 // Will jump to the compiled code just as if compiled code was doing it.
853 // Pre-load the register-jump target early, to schedule it better.
854 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_inline_offset())));
855
856 #if INCLUDE_JVMCI
857 if (EnableJVMCI) {
858 // check if this call should be routed towards a specific entry point
859 __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
860 Label no_alternative_target;
861 __ cbz(rscratch2, no_alternative_target);
862 __ mov(rscratch1, rscratch2);
863 __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
864 __ bind(no_alternative_target);
865 }
866 #endif // INCLUDE_JVMCI
867
868 int total_args_passed = sig->length();
869
870 // Now generate the shuffle code.
871 for (int i = 0; i < total_args_passed; i++) {
872 BasicType bt = sig->at(i)._bt;
873
874 assert(bt != T_INLINE_TYPE, "i2c adapter doesn't unpack inline typ args");
875 if (bt == T_VOID) {
876 assert(i > 0 && (sig->at(i - 1)._bt == T_LONG || sig->at(i - 1)._bt == T_DOUBLE), "missing half");
877 continue;
878 }
879
880 // Pick up 0, 1 or 2 words from SP+offset.
881 assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(), "scrambled load targets?");
882
883 // Load in argument order going down.
884 int ld_off = (total_args_passed - i - 1) * Interpreter::stackElementSize;
885 // Point to interpreter value (vs. tag)
886 int next_off = ld_off - Interpreter::stackElementSize;
887 //
888 //
889 //
890 VMReg r_1 = regs[i].first();
891 VMReg r_2 = regs[i].second();
892 if (!r_1->is_valid()) {
893 assert(!r_2->is_valid(), "");
894 continue;
895 }
896 if (r_1->is_stack()) {
897 // Convert stack slot to an SP offset (+ wordSize to account for return address )
898 int st_off = regs[i].first()->reg2stack() * VMRegImpl::stack_slot_size;
899 if (!r_2->is_valid()) {
900 // sign extend???
901 __ ldrsw(rscratch2, Address(esp, ld_off));
902 __ str(rscratch2, Address(sp, st_off));
903 } else {
904 //
905 // We are using two optoregs. This can be either T_OBJECT,
906 // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
907 // two slots but only uses one for thr T_LONG or T_DOUBLE case
908 // So we must adjust where to pick up the data to match the
909 // interpreter.
910 //
911 // Interpreter local[n] == MSW, local[n+1] == LSW however locals
912 // are accessed as negative so LSW is at LOW address
913
914 // ld_off is MSW so get LSW
915 const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;
916 __ ldr(rscratch2, Address(esp, offset));
917 // st_off is LSW (i.e. reg.first())
918 __ str(rscratch2, Address(sp, st_off));
919 }
920 } else if (r_1->is_Register()) { // Register argument
921 Register r = r_1->as_Register();
922 if (r_2->is_valid()) {
923 //
924 // We are using two VMRegs. This can be either T_OBJECT,
925 // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
926 // two slots but only uses one for thr T_LONG or T_DOUBLE case
927 // So we must adjust where to pick up the data to match the
928 // interpreter.
929
930 const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;
931
932 // this can be a misaligned move
933 __ ldr(r, Address(esp, offset));
934 } else {
935 // sign extend and use a full word?
936 __ ldrw(r, Address(esp, ld_off));
937 }
938 } else {
939 if (!r_2->is_valid()) {
940 __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
941 } else {
942 __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
943 }
944 }
945 }
946
947
948 // 6243940 We might end up in handle_wrong_method if
949 // the callee is deoptimized as we race thru here. If that
950 // happens we don't want to take a safepoint because the
951 // caller frame will look interpreted and arguments are now
952 // "compiled" so it is much better to make this transition
953 // invisible to the stack walking code. Unfortunately if
954 // we try and find the callee by normal means a safepoint
955 // is possible. So we stash the desired callee in the thread
956 // and the vm will find there should this case occur.
957
958 __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));
959 __ br(rscratch1);
960 }
961
962 static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) {
963
964 Label ok;
965
966 Register holder = rscratch2;
967 Register receiver = j_rarg0;
968 Register tmp = r10; // A call-clobbered register not used for arg passing
969
970 // -------------------------------------------------------------------------
971 // Generate a C2I adapter. On entry we know rmethod holds the Method* during calls
972 // to the interpreter. The args start out packed in the compiled layout. They
973 // need to be unpacked into the interpreter layout. This will almost always
974 // require some stack space. We grow the current (compiled) stack, then repack
975 // the args. We finally end in a jump to the generic interpreter entry point.
976 // On exit from the interpreter, the interpreter will restore our SP (lest the
977 // compiled code, which relys solely on SP and not FP, get sick).
978
979 {
980 __ block_comment("c2i_unverified_entry {");
981 __ load_klass(rscratch1, receiver);
982 __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
983 __ cmp(rscratch1, tmp);
984 __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
985 __ br(Assembler::EQ, ok);
986 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
987
988 __ bind(ok);
989 // Method might have been compiled since the call site was patched to
990 // interpreted; if that is the case treat it as a miss so we can get
991 // the call site corrected.
992 __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
993 __ cbz(rscratch1, skip_fixup);
994 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
995 __ block_comment("} c2i_unverified_entry");
996 }
997 }
998
999
1000 // ---------------------------------------------------------------
1001 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,
1002 int comp_args_on_stack,
1003 const GrowableArray<SigEntry>* sig,
1004 const VMRegPair* regs,
1005 const GrowableArray<SigEntry>* sig_cc,
1006 const VMRegPair* regs_cc,
1007 const GrowableArray<SigEntry>* sig_cc_ro,
1008 const VMRegPair* regs_cc_ro,
1009 AdapterFingerPrint* fingerprint,
1010 AdapterBlob*& new_adapter,
1011 bool allocate_code_blob) {
1012
1013 address i2c_entry = __ pc();
1014 gen_i2c_adapter(masm, comp_args_on_stack, sig, regs);
1015
1016 address c2i_unverified_entry = __ pc();
1017 Label skip_fixup;
1018
1019 gen_inline_cache_check(masm, skip_fixup);
1020
1021 OopMapSet* oop_maps = new OopMapSet();
1022 int frame_complete = CodeOffsets::frame_never_safe;
1023 int frame_size_in_words = 0;
1024
1025 // Scalarized c2i adapter with non-scalarized receiver (i.e., don't pack receiver)
1026 address c2i_inline_ro_entry = __ pc();
1027 if (regs_cc != regs_cc_ro) {
1028 gen_c2i_adapter(masm, sig_cc_ro, regs_cc_ro, skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, false);
1029 skip_fixup.reset();
1030 }
1031
1032 // Scalarized c2i adapter
1033 address c2i_entry = __ pc();
1034
1035 // Class initialization barrier for static methods
1036 address c2i_no_clinit_check_entry = NULL;
1037 if (VM_Version::supports_fast_class_init_checks()) {
1038 Label L_skip_barrier;
1039
1040 { // Bypass the barrier for non-static methods
1041 __ ldrw(rscratch1, Address(rmethod, Method::access_flags_offset()));
1042 __ andsw(zr, rscratch1, JVM_ACC_STATIC);
1043 __ br(Assembler::EQ, L_skip_barrier); // non-static
1044 }
1045
1046 __ load_method_holder(rscratch2, rmethod);
1047 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1048 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1049
1050 __ bind(L_skip_barrier);
1051 c2i_no_clinit_check_entry = __ pc();
1052 }
1053
1054 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1055 bs->c2i_entry_barrier(masm);
1056
1057 gen_c2i_adapter(masm, sig_cc, regs_cc, skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, true);
1058
1059 address c2i_unverified_inline_entry = c2i_unverified_entry;
1060
1061 // Non-scalarized c2i adapter
1062 address c2i_inline_entry = c2i_entry;
1063 if (regs != regs_cc) {
1064 Label inline_entry_skip_fixup;
1065 c2i_unverified_inline_entry = __ pc();
1066 gen_inline_cache_check(masm, inline_entry_skip_fixup);
1067
1068 c2i_inline_entry = __ pc();
1069 gen_c2i_adapter(masm, sig, regs, inline_entry_skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, false);
1070 }
1071
1072 __ flush();
1073
1074 // The c2i adapter might safepoint and trigger a GC. The caller must make sure that
1075 // the GC knows about the location of oop argument locations passed to the c2i adapter.
1076 if (allocate_code_blob) {
1077 bool caller_must_gc_arguments = (regs != regs_cc);
1078 new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
1079 }
1080
1081 return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry, c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
1082 }
1083
1084 static int c_calling_convention_priv(const BasicType *sig_bt,
1085 VMRegPair *regs,
1086 VMRegPair *regs2,
1087 int total_args_passed) {
1088 assert(regs2 == NULL, "not needed on AArch64");
1089
1090 // We return the amount of VMRegImpl stack slots we need to reserve for all
1091 // the arguments NOT counting out_preserve_stack_slots.
1092
1093 static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1094 c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5, c_rarg6, c_rarg7
1095 };
1096 static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
1097 c_farg0, c_farg1, c_farg2, c_farg3,
1098 c_farg4, c_farg5, c_farg6, c_farg7
1099 };
1100
1101 uint int_args = 0;
1109 case T_BYTE:
1110 case T_SHORT:
1111 case T_INT:
1112 if (int_args < Argument::n_int_register_parameters_c) {
1113 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
1114 } else {
1115 #ifdef __APPLE__
1116 // Less-than word types are stored one after another.
1117 // The code is unable to handle this so bailout.
1118 return -1;
1119 #endif
1120 regs[i].set1(VMRegImpl::stack2reg(stk_args));
1121 stk_args += 2;
1122 }
1123 break;
1124 case T_LONG:
1125 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
1126 // fall through
1127 case T_OBJECT:
1128 case T_ARRAY:
1129 case T_INLINE_TYPE:
1130 case T_ADDRESS:
1131 case T_METADATA:
1132 if (int_args < Argument::n_int_register_parameters_c) {
1133 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
1134 } else {
1135 regs[i].set2(VMRegImpl::stack2reg(stk_args));
1136 stk_args += 2;
1137 }
1138 break;
1139 case T_FLOAT:
1140 if (fp_args < Argument::n_float_register_parameters_c) {
1141 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
1142 } else {
1143 #ifdef __APPLE__
1144 // Less-than word types are stored one after another.
1145 // The code is unable to handle this so bailout.
1146 return -1;
1147 #endif
1148 regs[i].set1(VMRegImpl::stack2reg(stk_args));
1149 stk_args += 2;
1784 int temploc = -1;
1785 for (int ai = 0; ai < arg_order.length(); ai += 2) {
1786 int i = arg_order.at(ai);
1787 int c_arg = arg_order.at(ai + 1);
1788 __ block_comment(err_msg("move %d -> %d", i, c_arg));
1789 assert(c_arg != -1 && i != -1, "wrong order");
1790 #ifdef ASSERT
1791 if (in_regs[i].first()->is_Register()) {
1792 assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
1793 } else if (in_regs[i].first()->is_FloatRegister()) {
1794 assert(!freg_destroyed[in_regs[i].first()->as_FloatRegister()->encoding()], "destroyed reg!");
1795 }
1796 if (out_regs[c_arg].first()->is_Register()) {
1797 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1798 } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1799 freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1800 }
1801 #endif /* ASSERT */
1802 switch (in_sig_bt[i]) {
1803 case T_ARRAY:
1804 case T_INLINE_TYPE:
1805 case T_OBJECT:
1806 object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
1807 ((i == 0) && (!is_static)),
1808 &receiver_offset);
1809 int_args++;
1810 break;
1811 case T_VOID:
1812 break;
1813
1814 case T_FLOAT:
1815 float_move(masm, in_regs[i], out_regs[c_arg]);
1816 float_args++;
1817 break;
1818
1819 case T_DOUBLE:
1820 assert( i + 1 < total_in_args &&
1821 in_sig_bt[i + 1] == T_VOID &&
1822 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
1823 double_move(masm, in_regs[i], out_regs[c_arg]);
1824 float_args++;
1899 Label slow_path_lock;
1900 Label lock_done;
1901
1902 if (method->is_synchronized()) {
1903
1904 const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
1905
1906 // Get the handle (the 2nd argument)
1907 __ mov(oop_handle_reg, c_rarg1);
1908
1909 // Get address of the box
1910
1911 __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1912
1913 // Load the oop from the handle
1914 __ ldr(obj_reg, Address(oop_handle_reg, 0));
1915
1916 // Load (object->mark() | 1) into swap_reg %r0
1917 __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1918 __ orr(swap_reg, rscratch1, 1);
1919 if (EnableValhalla) {
1920 // Mask inline_type bit such that we go to the slow path if object is an inline type
1921 __ andr(swap_reg, swap_reg, ~((int) markWord::inline_type_bit_in_place));
1922 }
1923
1924 // Save (object->mark() | 1) into BasicLock's displaced header
1925 __ str(swap_reg, Address(lock_reg, mark_word_offset));
1926
1927 // src -> dest iff dest == r0 else r0 <- dest
1928 { Label here;
1929 __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, lock_done, /*fallthrough*/NULL);
1930 }
1931
1932 // Hmm should this move to the slow path code area???
1933
1934 // Test if the oopMark is an obvious stack pointer, i.e.,
1935 // 1) (mark & 3) == 0, and
1936 // 2) sp <= mark < mark + os::pagesize()
1937 // These 3 tests can be done by evaluating the following
1938 // expression: ((mark - sp) & (3 - os::vm_page_size())),
1939 // assuming both stack pointer and pagesize have their
1940 // least significant 2 bits clear.
1941 // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
1942
1966
1967 rt_call(masm, native_func);
1968
1969 __ bind(native_return);
1970
1971 intptr_t return_pc = (intptr_t) __ pc();
1972 oop_maps->add_gc_map(return_pc - start, map);
1973
1974 // Unpack native results.
1975 switch (ret_type) {
1976 case T_BOOLEAN: __ c2bool(r0); break;
1977 case T_CHAR : __ ubfx(r0, r0, 0, 16); break;
1978 case T_BYTE : __ sbfx(r0, r0, 0, 8); break;
1979 case T_SHORT : __ sbfx(r0, r0, 0, 16); break;
1980 case T_INT : __ sbfx(r0, r0, 0, 32); break;
1981 case T_DOUBLE :
1982 case T_FLOAT :
1983 // Result is in v0 we'll save as needed
1984 break;
1985 case T_ARRAY: // Really a handle
1986 case T_INLINE_TYPE: // Really a handle
1987 case T_OBJECT: // Really a handle
1988 break; // can't de-handlize until after safepoint check
1989 case T_VOID: break;
1990 case T_LONG: break;
1991 default : ShouldNotReachHere();
1992 }
1993
1994 Label safepoint_in_progress, safepoint_in_progress_done;
1995 Label after_transition;
1996
1997 // Switch thread to "native transition" state before reading the synchronization state.
1998 // This additional state is necessary because reading and testing the synchronization
1999 // state is not atomic w.r.t. GC, as this scenario demonstrates:
2000 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2001 // VM thread changes sync state to synchronizing and suspends threads for GC.
2002 // Thread A is resumed to finish this native method, but doesn't block here since it
2003 // didn't see any synchronization is progress, and escapes.
2004 __ mov(rscratch1, _thread_in_native_trans);
2005
2006 __ strw(rscratch1, Address(rthread, JavaThread::thread_state_offset()));
3191 #ifdef ASSERT
3192 __ str(zr, Address(rthread, JavaThread::exception_handler_pc_offset()));
3193 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
3194 #endif
3195 // Clear the exception oop so GC no longer processes it as a root.
3196 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
3197
3198 // r0: exception oop
3199 // r8: exception handler
3200 // r4: exception pc
3201 // Jump to handler
3202
3203 __ br(r8);
3204
3205 // Make sure all code is generated
3206 masm->flush();
3207
3208 // Set exception blob
3209 _exception_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
3210 }
3211 #endif // COMPILER2
3212
3213 BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) {
3214 BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K);
3215 CodeBuffer buffer(buf);
3216 short buffer_locs[20];
3217 buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
3218 sizeof(buffer_locs)/sizeof(relocInfo));
3219
3220 MacroAssembler _masm(&buffer);
3221 MacroAssembler* masm = &_masm;
3222
3223 const Array<SigEntry>* sig_vk = vk->extended_sig();
3224 const Array<VMRegPair>* regs = vk->return_regs();
3225
3226 int pack_fields_jobject_off = __ offset();
3227 // Resolve pre-allocated buffer from JNI handle.
3228 // We cannot do this in generate_call_stub() because it requires GC code to be initialized.
3229 Register Rresult = r14; // See StubGenerator::generate_call_stub().
3230 __ ldr(r0, Address(Rresult));
3231 __ resolve_jobject(r0 /* value */,
3232 rthread /* thread */,
3233 r12 /* tmp */);
3234 __ str(r0, Address(Rresult));
3235
3236 int pack_fields_off = __ offset();
3237
3238 int j = 1;
3239 for (int i = 0; i < sig_vk->length(); i++) {
3240 BasicType bt = sig_vk->at(i)._bt;
3241 if (bt == T_INLINE_TYPE) {
3242 continue;
3243 }
3244 if (bt == T_VOID) {
3245 if (sig_vk->at(i-1)._bt == T_LONG ||
3246 sig_vk->at(i-1)._bt == T_DOUBLE) {
3247 j++;
3248 }
3249 continue;
3250 }
3251 int off = sig_vk->at(i)._offset;
3252 VMRegPair pair = regs->at(j);
3253 VMReg r_1 = pair.first();
3254 VMReg r_2 = pair.second();
3255 Address to(r0, off);
3256 if (bt == T_FLOAT) {
3257 __ strs(r_1->as_FloatRegister(), to);
3258 } else if (bt == T_DOUBLE) {
3259 __ strd(r_1->as_FloatRegister(), to);
3260 } else if (bt == T_OBJECT || bt == T_ARRAY) {
3261 Register val = r_1->as_Register();
3262 assert_different_registers(r0, val);
3263 // We don't need barriers because the destination is a newly allocated object.
3264 // Also, we cannot use store_heap_oop(to, val) because it uses r8 as tmp.
3265 if (UseCompressedOops) {
3266 __ encode_heap_oop(val);
3267 __ str(val, to);
3268 } else {
3269 __ str(val, to);
3270 }
3271 } else {
3272 assert(is_java_primitive(bt), "unexpected basic type");
3273 assert_different_registers(r0, r_1->as_Register());
3274 size_t size_in_bytes = type2aelembytes(bt);
3275 __ store_sized_value(to, r_1->as_Register(), size_in_bytes);
3276 }
3277 j++;
3278 }
3279 assert(j == regs->length(), "missed a field?");
3280
3281 __ ret(lr);
3282
3283 int unpack_fields_off = __ offset();
3284
3285 j = 1;
3286 for (int i = 0; i < sig_vk->length(); i++) {
3287 BasicType bt = sig_vk->at(i)._bt;
3288 if (bt == T_INLINE_TYPE) {
3289 continue;
3290 }
3291 if (bt == T_VOID) {
3292 if (sig_vk->at(i-1)._bt == T_LONG ||
3293 sig_vk->at(i-1)._bt == T_DOUBLE) {
3294 j++;
3295 }
3296 continue;
3297 }
3298 int off = sig_vk->at(i)._offset;
3299 assert(off > 0, "offset in object should be positive");
3300 VMRegPair pair = regs->at(j);
3301 VMReg r_1 = pair.first();
3302 VMReg r_2 = pair.second();
3303 Address from(r0, off);
3304 if (bt == T_FLOAT) {
3305 __ ldrs(r_1->as_FloatRegister(), from);
3306 } else if (bt == T_DOUBLE) {
3307 __ ldrd(r_1->as_FloatRegister(), from);
3308 } else if (bt == T_OBJECT || bt == T_ARRAY) {
3309 assert_different_registers(r0, r_1->as_Register());
3310 __ load_heap_oop(r_1->as_Register(), from);
3311 } else {
3312 assert(is_java_primitive(bt), "unexpected basic type");
3313 assert_different_registers(r0, r_1->as_Register());
3314
3315 size_t size_in_bytes = type2aelembytes(bt);
3316 __ load_sized_value(r_1->as_Register(), from, size_in_bytes, bt != T_CHAR && bt != T_BOOLEAN);
3317 }
3318 j++;
3319 }
3320 assert(j == regs->length(), "missed a field?");
3321
3322 if (StressInlineTypeReturnedAsFields) {
3323 __ load_klass(r0, r0);
3324 __ orr(r0, r0, 1);
3325 }
3326
3327 __ ret(lr);
3328
3329 __ flush();
3330
3331 return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
3332 }
3333
3334 // ---------------------------------------------------------------
3335
3336 class NativeInvokerGenerator : public StubCodeGenerator {
3337 address _call_target;
3338 int _shadow_space_bytes;
3339
3340 const GrowableArray<VMReg>& _input_registers;
3341 const GrowableArray<VMReg>& _output_registers;
3342
3343 int _frame_complete;
3344 int _framesize;
3345 OopMapSet* _oop_maps;
3346 public:
3347 NativeInvokerGenerator(CodeBuffer* buffer,
3348 address call_target,
3349 int shadow_space_bytes,
3350 const GrowableArray<VMReg>& input_registers,
3351 const GrowableArray<VMReg>& output_registers)
3352 : StubCodeGenerator(buffer, PrintMethodHandleStubs),
3565
3566 //////////////////////////////////////////////////////////////////////////////
3567
3568 __ block_comment("{ L_reguard");
3569 __ bind(L_reguard);
3570
3571 spill_output_registers();
3572
3573 rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
3574
3575 fill_output_registers();
3576
3577 __ b(L_after_reguard);
3578
3579 __ block_comment("} L_reguard");
3580
3581 //////////////////////////////////////////////////////////////////////////////
3582
3583 __ flush();
3584 }
|