23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "nativeInst_aarch64.hpp"
32 #include "oops/instanceOop.hpp"
33 #include "oops/method.hpp"
34 #include "oops/objArrayKlass.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "prims/methodHandles.hpp"
37 #include "runtime/frame.inline.hpp"
38 #include "runtime/handles.inline.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "runtime/stubCodeGenerator.hpp"
41 #include "runtime/stubRoutines.hpp"
42 #include "runtime/thread.inline.hpp"
43 #include "utilities/top.hpp"
44 #ifdef COMPILER2
45 #include "opto/runtime.hpp"
46 #endif
47
48 // Declaration and definition of StubGenerator (no .hpp file).
49 // For a more detailed description of the stub routine structure
50 // see the comment in stubRoutines.hpp
51
52 #undef __
53 #define __ _masm->
54 #define TIMES_OOP Address::sxtw(exact_log2(UseCompressedOops ? 4 : 8))
55
56 #ifdef PRODUCT
57 #define BLOCK_COMMENT(str) /* nothing */
58 #else
59 #define BLOCK_COMMENT(str) __ block_comment(str)
60 #endif
61
62 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
63
64 // Stub Code definitions
65
66 class StubGenerator: public StubCodeGenerator {
580 #ifndef PRODUCT
581 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
582 #endif
583 BLOCK_COMMENT("call MacroAssembler::debug");
584 __ mov(rscratch1, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
585 __ blr(rscratch1);
586
587 return start;
588 }
589
590 void array_overlap_test(Label& L_no_overlap, Address::sxtw sf) { __ b(L_no_overlap); }
591
592 // Generate code for an array write pre barrier
593 //
594 // addr - starting address
595 // count - element count
596 // tmp - scratch register
597 //
598 // Destroy no registers except rscratch1 and rscratch2
599 //
600 void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) {
601 BarrierSet* bs = Universe::heap()->barrier_set();
602 switch (bs->kind()) {
603 case BarrierSet::G1SATBCT:
604 case BarrierSet::G1SATBCTLogging:
605 // With G1, don't generate the call if we statically know that the target in uninitialized
606 if (!dest_uninitialized) {
607 __ push_call_clobbered_registers();
608 if (count == c_rarg0) {
609 if (addr == c_rarg1) {
610 // exactly backwards!!
611 __ mov(rscratch1, c_rarg0);
612 __ mov(c_rarg0, c_rarg1);
613 __ mov(c_rarg1, rscratch1);
614 } else {
615 __ mov(c_rarg1, count);
616 __ mov(c_rarg0, addr);
617 }
618 } else {
619 __ mov(c_rarg0, addr);
620 __ mov(c_rarg1, count);
621 }
622 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
623 __ pop_call_clobbered_registers();
624 break;
625 case BarrierSet::CardTableModRef:
626 case BarrierSet::CardTableExtension:
627 case BarrierSet::ModRef:
628 break;
629 default:
630 ShouldNotReachHere();
631
632 }
633 }
634 }
635
636 //
637 // Generate code for an array write post barrier
638 //
639 // Input:
640 // start - register containing starting address of destination array
641 // end - register containing ending address of destination array
642 // scratch - scratch register
643 //
644 // The input registers are overwritten.
645 // The ending address is inclusive.
646 void gen_write_ref_array_post_barrier(Register start, Register end, Register scratch) {
647 assert_different_registers(start, end, scratch);
648 Label L_done;
677 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
678
679 Label L_loop;
680
681 __ lsr(start, start, CardTableModRefBS::card_shift);
682 __ lsr(end, end, CardTableModRefBS::card_shift);
683 __ sub(end, end, start); // number of bytes to copy
684
685 const Register count = end; // 'end' register contains bytes count now
686 __ load_byte_map_base(scratch);
687 __ add(start, start, scratch);
688 if (UseConcMarkSweepGC) {
689 __ membar(__ StoreStore);
690 }
691 __ BIND(L_loop);
692 __ strb(zr, Address(start, count));
693 __ subs(count, count, 1);
694 __ br(Assembler::GE, L_loop);
695 }
696 break;
697 default:
698 ShouldNotReachHere();
699
700 }
701 __ bind(L_done);
702 }
703
704 address generate_zero_longs(Register base, Register cnt) {
705 Register tmp = rscratch1;
706 Register tmp2 = rscratch2;
707 int zva_length = VM_Version::zva_length();
708 Label initial_table_end, loop_zva;
709 Label fini;
710
711 __ align(CodeEntryAlignment);
712 StubCodeMark mark(this, "StubRoutines", "zero_longs");
713 address start = __ pc();
714
715 // Base must be 16 byte aligned. If not just return and let caller handle it
716 __ tst(base, 0x0f);
1394 // disjoint_int_copy_entry is set to the no-overlap entry point
1395 // used by generate_conjoint_int_oop_copy().
1396 //
1397 address generate_disjoint_copy(size_t size, bool aligned, bool is_oop, address *entry,
1398 const char *name, bool dest_uninitialized = false) {
1399 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1400 __ align(CodeEntryAlignment);
1401 StubCodeMark mark(this, "StubRoutines", name);
1402 address start = __ pc();
1403 __ enter();
1404
1405 if (entry != NULL) {
1406 *entry = __ pc();
1407 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1408 BLOCK_COMMENT("Entry:");
1409 }
1410
1411 if (is_oop) {
1412 __ push(RegSet::of(d, count), sp);
1413 // no registers are destroyed by this call
1414 gen_write_ref_array_pre_barrier(d, count, dest_uninitialized);
1415 }
1416 copy_memory(aligned, s, d, count, rscratch1, size);
1417 if (is_oop) {
1418 __ pop(RegSet::of(d, count), sp);
1419 if (VerifyOops)
1420 verify_oop_array(size, d, count, r16);
1421 __ sub(count, count, 1); // make an inclusive end pointer
1422 __ lea(count, Address(d, count, Address::lsl(exact_log2(size))));
1423 gen_write_ref_array_post_barrier(d, count, rscratch1);
1424 }
1425 __ leave();
1426 __ mov(r0, zr); // return 0
1427 __ ret(lr);
1428 return start;
1429 }
1430
1431 // Arguments:
1432 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1433 // ignored
1434 // is_oop - true => oop array, so generate store check code
1450
1451 StubCodeMark mark(this, "StubRoutines", name);
1452 address start = __ pc();
1453
1454 __ enter();
1455
1456 if (entry != NULL) {
1457 *entry = __ pc();
1458 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1459 BLOCK_COMMENT("Entry:");
1460 }
1461
1462 // use fwd copy when (d-s) above_equal (count*size)
1463 __ sub(rscratch1, d, s);
1464 __ cmp(rscratch1, count, Assembler::LSL, exact_log2(size));
1465 __ br(Assembler::HS, nooverlap_target);
1466
1467 if (is_oop) {
1468 __ push(RegSet::of(d, count), sp);
1469 // no registers are destroyed by this call
1470 gen_write_ref_array_pre_barrier(d, count, dest_uninitialized);
1471 }
1472 copy_memory(aligned, s, d, count, rscratch1, -size);
1473 if (is_oop) {
1474 __ pop(RegSet::of(d, count), sp);
1475 if (VerifyOops)
1476 verify_oop_array(size, d, count, r16);
1477 __ sub(count, count, 1); // make an inclusive end pointer
1478 __ lea(count, Address(d, count, Address::lsl(exact_log2(size))));
1479 gen_write_ref_array_post_barrier(d, count, rscratch1);
1480 }
1481 __ leave();
1482 __ mov(r0, zr); // return 0
1483 __ ret(lr);
1484 return start;
1485 }
1486
1487 // Arguments:
1488 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1489 // ignored
1490 // name - stub name string
1584 }
1585 // Arguments:
1586 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1587 // ignored
1588 // name - stub name string
1589 //
1590 // Inputs:
1591 // c_rarg0 - source array address
1592 // c_rarg1 - destination array address
1593 // c_rarg2 - element count, treated as ssize_t, can be zero
1594 //
1595 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1596 // the hardware handle it. The two dwords within qwords that span
1597 // cache line boundaries will still be loaded and stored atomicly.
1598 //
1599 // Side Effects:
1600 // disjoint_int_copy_entry is set to the no-overlap entry point
1601 // used by generate_conjoint_int_oop_copy().
1602 //
1603 address generate_disjoint_int_copy(bool aligned, address *entry,
1604 const char *name, bool dest_uninitialized = false) {
1605 const bool not_oop = false;
1606 return generate_disjoint_copy(sizeof (jint), aligned, not_oop, entry, name);
1607 }
1608
1609 // Arguments:
1610 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1611 // ignored
1612 // name - stub name string
1613 //
1614 // Inputs:
1615 // c_rarg0 - source array address
1616 // c_rarg1 - destination array address
1617 // c_rarg2 - element count, treated as ssize_t, can be zero
1618 //
1619 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1620 // the hardware handle it. The two dwords within qwords that span
1621 // cache line boundaries will still be loaded and stored atomicly.
1622 //
1623 address generate_conjoint_int_copy(bool aligned, address nooverlap_target,
1624 address *entry, const char *name,
1792
1793 // Empty array: Nothing to do.
1794 __ cbz(count, L_done);
1795
1796 __ push(RegSet::of(r18, r19, r20, r21), sp);
1797
1798 #ifdef ASSERT
1799 BLOCK_COMMENT("assert consistent ckoff/ckval");
1800 // The ckoff and ckval must be mutually consistent,
1801 // even though caller generates both.
1802 { Label L;
1803 int sco_offset = in_bytes(Klass::super_check_offset_offset());
1804 __ ldrw(start_to, Address(ckval, sco_offset));
1805 __ cmpw(ckoff, start_to);
1806 __ br(Assembler::EQ, L);
1807 __ stop("super_check_offset inconsistent");
1808 __ bind(L);
1809 }
1810 #endif //ASSERT
1811
1812 gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
1813
1814 // save the original count
1815 __ mov(count_save, count);
1816
1817 // Copy from low to high addresses
1818 __ mov(start_to, to); // Save destination array start address
1819 __ b(L_load_element);
1820
1821 // ======== begin loop ========
1822 // (Loop is rotated; its entry is L_load_element.)
1823 // Loop control:
1824 // for (; count != 0; count--) {
1825 // copied_oop = load_heap_oop(from++);
1826 // ... generate_type_check ...;
1827 // store_heap_oop(to++, copied_oop);
1828 // }
1829 __ align(OptoLoopAlignment);
1830
1831 __ BIND(L_store_element);
1832 __ store_heap_oop(__ post(to, UseCompressedOops ? 4 : 8), copied_oop); // store the oop
|
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "nativeInst_aarch64.hpp"
32 #include "oops/instanceOop.hpp"
33 #include "oops/method.hpp"
34 #include "oops/objArrayKlass.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "prims/methodHandles.hpp"
37 #include "runtime/frame.inline.hpp"
38 #include "runtime/handles.inline.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "runtime/stubCodeGenerator.hpp"
41 #include "runtime/stubRoutines.hpp"
42 #include "runtime/thread.inline.hpp"
43 #include "utilities/macros.hpp"
44 #include "utilities/top.hpp"
45
46 #include "stubRoutines_aarch64.hpp"
47
48 #ifdef COMPILER2
49 #include "opto/runtime.hpp"
50 #endif
51 #if INCLUDE_ALL_GCS
52 #include "shenandoahBarrierSetAssembler_aarch64.hpp"
53 #endif
54
55 // Declaration and definition of StubGenerator (no .hpp file).
56 // For a more detailed description of the stub routine structure
57 // see the comment in stubRoutines.hpp
58
59 #undef __
60 #define __ _masm->
61 #define TIMES_OOP Address::sxtw(exact_log2(UseCompressedOops ? 4 : 8))
62
63 #ifdef PRODUCT
64 #define BLOCK_COMMENT(str) /* nothing */
65 #else
66 #define BLOCK_COMMENT(str) __ block_comment(str)
67 #endif
68
69 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
70
71 // Stub Code definitions
72
73 class StubGenerator: public StubCodeGenerator {
587 #ifndef PRODUCT
588 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
589 #endif
590 BLOCK_COMMENT("call MacroAssembler::debug");
591 __ mov(rscratch1, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
592 __ blr(rscratch1);
593
594 return start;
595 }
596
597 void array_overlap_test(Label& L_no_overlap, Address::sxtw sf) { __ b(L_no_overlap); }
598
599 // Generate code for an array write pre barrier
600 //
601 // addr - starting address
602 // count - element count
603 // tmp - scratch register
604 //
605 // Destroy no registers except rscratch1 and rscratch2
606 //
607 void gen_write_ref_array_pre_barrier(Register src, Register addr, Register count, bool dest_uninitialized) {
608 BarrierSet* bs = Universe::heap()->barrier_set();
609 switch (bs->kind()) {
610 case BarrierSet::G1SATBCT:
611 case BarrierSet::G1SATBCTLogging:
612 // Don't generate the call if we statically know that the target is uninitialized
613 if (!dest_uninitialized) {
614 __ push_call_clobbered_registers();
615 if (count == c_rarg0) {
616 if (addr == c_rarg1) {
617 // exactly backwards!!
618 __ mov(rscratch1, c_rarg0);
619 __ mov(c_rarg0, c_rarg1);
620 __ mov(c_rarg1, rscratch1);
621 } else {
622 __ mov(c_rarg1, count);
623 __ mov(c_rarg0, addr);
624 }
625 } else {
626 __ mov(c_rarg0, addr);
627 __ mov(c_rarg1, count);
628 }
629 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
630 __ pop_call_clobbered_registers();
631 break;
632 case BarrierSet::CardTableModRef:
633 case BarrierSet::CardTableExtension:
634 case BarrierSet::ModRef:
635 break;
636 #if INCLUDE_ALL_GCS
637 case BarrierSet::ShenandoahBarrierSet:
638 ShenandoahBarrierSetAssembler::bsasm()->arraycopy_prologue(_masm, dest_uninitialized, src, addr, count);
639 break;
640 #endif
641 default:
642 ShouldNotReachHere();
643
644 }
645 }
646 }
647
648 //
649 // Generate code for an array write post barrier
650 //
651 // Input:
652 // start - register containing starting address of destination array
653 // end - register containing ending address of destination array
654 // scratch - scratch register
655 //
656 // The input registers are overwritten.
657 // The ending address is inclusive.
658 void gen_write_ref_array_post_barrier(Register start, Register end, Register scratch) {
659 assert_different_registers(start, end, scratch);
660 Label L_done;
689 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
690
691 Label L_loop;
692
693 __ lsr(start, start, CardTableModRefBS::card_shift);
694 __ lsr(end, end, CardTableModRefBS::card_shift);
695 __ sub(end, end, start); // number of bytes to copy
696
697 const Register count = end; // 'end' register contains bytes count now
698 __ load_byte_map_base(scratch);
699 __ add(start, start, scratch);
700 if (UseConcMarkSweepGC) {
701 __ membar(__ StoreStore);
702 }
703 __ BIND(L_loop);
704 __ strb(zr, Address(start, count));
705 __ subs(count, count, 1);
706 __ br(Assembler::GE, L_loop);
707 }
708 break;
709 #if INCLUDE_ALL_GCS
710 case BarrierSet::ShenandoahBarrierSet:
711 break;
712 #endif
713 default:
714 ShouldNotReachHere();
715
716 }
717 __ bind(L_done);
718 }
719
720 address generate_zero_longs(Register base, Register cnt) {
721 Register tmp = rscratch1;
722 Register tmp2 = rscratch2;
723 int zva_length = VM_Version::zva_length();
724 Label initial_table_end, loop_zva;
725 Label fini;
726
727 __ align(CodeEntryAlignment);
728 StubCodeMark mark(this, "StubRoutines", "zero_longs");
729 address start = __ pc();
730
731 // Base must be 16 byte aligned. If not just return and let caller handle it
732 __ tst(base, 0x0f);
1410 // disjoint_int_copy_entry is set to the no-overlap entry point
1411 // used by generate_conjoint_int_oop_copy().
1412 //
1413 address generate_disjoint_copy(size_t size, bool aligned, bool is_oop, address *entry,
1414 const char *name, bool dest_uninitialized = false) {
1415 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1416 __ align(CodeEntryAlignment);
1417 StubCodeMark mark(this, "StubRoutines", name);
1418 address start = __ pc();
1419 __ enter();
1420
1421 if (entry != NULL) {
1422 *entry = __ pc();
1423 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1424 BLOCK_COMMENT("Entry:");
1425 }
1426
1427 if (is_oop) {
1428 __ push(RegSet::of(d, count), sp);
1429 // no registers are destroyed by this call
1430 gen_write_ref_array_pre_barrier(s, d, count, dest_uninitialized);
1431 }
1432 copy_memory(aligned, s, d, count, rscratch1, size);
1433 if (is_oop) {
1434 __ pop(RegSet::of(d, count), sp);
1435 if (VerifyOops)
1436 verify_oop_array(size, d, count, r16);
1437 __ sub(count, count, 1); // make an inclusive end pointer
1438 __ lea(count, Address(d, count, Address::lsl(exact_log2(size))));
1439 gen_write_ref_array_post_barrier(d, count, rscratch1);
1440 }
1441 __ leave();
1442 __ mov(r0, zr); // return 0
1443 __ ret(lr);
1444 return start;
1445 }
1446
1447 // Arguments:
1448 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1449 // ignored
1450 // is_oop - true => oop array, so generate store check code
1466
1467 StubCodeMark mark(this, "StubRoutines", name);
1468 address start = __ pc();
1469
1470 __ enter();
1471
1472 if (entry != NULL) {
1473 *entry = __ pc();
1474 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1475 BLOCK_COMMENT("Entry:");
1476 }
1477
1478 // use fwd copy when (d-s) above_equal (count*size)
1479 __ sub(rscratch1, d, s);
1480 __ cmp(rscratch1, count, Assembler::LSL, exact_log2(size));
1481 __ br(Assembler::HS, nooverlap_target);
1482
1483 if (is_oop) {
1484 __ push(RegSet::of(d, count), sp);
1485 // no registers are destroyed by this call
1486 gen_write_ref_array_pre_barrier(s, d, count, dest_uninitialized);
1487 }
1488 copy_memory(aligned, s, d, count, rscratch1, -size);
1489 if (is_oop) {
1490 __ pop(RegSet::of(d, count), sp);
1491 if (VerifyOops)
1492 verify_oop_array(size, d, count, r16);
1493 __ sub(count, count, 1); // make an inclusive end pointer
1494 __ lea(count, Address(d, count, Address::lsl(exact_log2(size))));
1495 gen_write_ref_array_post_barrier(d, count, rscratch1);
1496 }
1497 __ leave();
1498 __ mov(r0, zr); // return 0
1499 __ ret(lr);
1500 return start;
1501 }
1502
1503 // Arguments:
1504 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1505 // ignored
1506 // name - stub name string
1600 }
1601 // Arguments:
1602 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1603 // ignored
1604 // name - stub name string
1605 //
1606 // Inputs:
1607 // c_rarg0 - source array address
1608 // c_rarg1 - destination array address
1609 // c_rarg2 - element count, treated as ssize_t, can be zero
1610 //
1611 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1612 // the hardware handle it. The two dwords within qwords that span
1613 // cache line boundaries will still be loaded and stored atomicly.
1614 //
1615 // Side Effects:
1616 // disjoint_int_copy_entry is set to the no-overlap entry point
1617 // used by generate_conjoint_int_oop_copy().
1618 //
1619 address generate_disjoint_int_copy(bool aligned, address *entry,
1620 const char *name) {
1621 const bool not_oop = false;
1622 return generate_disjoint_copy(sizeof (jint), aligned, not_oop, entry, name);
1623 }
1624
1625 // Arguments:
1626 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1627 // ignored
1628 // name - stub name string
1629 //
1630 // Inputs:
1631 // c_rarg0 - source array address
1632 // c_rarg1 - destination array address
1633 // c_rarg2 - element count, treated as ssize_t, can be zero
1634 //
1635 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1636 // the hardware handle it. The two dwords within qwords that span
1637 // cache line boundaries will still be loaded and stored atomicly.
1638 //
1639 address generate_conjoint_int_copy(bool aligned, address nooverlap_target,
1640 address *entry, const char *name,
1808
1809 // Empty array: Nothing to do.
1810 __ cbz(count, L_done);
1811
1812 __ push(RegSet::of(r18, r19, r20, r21), sp);
1813
1814 #ifdef ASSERT
1815 BLOCK_COMMENT("assert consistent ckoff/ckval");
1816 // The ckoff and ckval must be mutually consistent,
1817 // even though caller generates both.
1818 { Label L;
1819 int sco_offset = in_bytes(Klass::super_check_offset_offset());
1820 __ ldrw(start_to, Address(ckval, sco_offset));
1821 __ cmpw(ckoff, start_to);
1822 __ br(Assembler::EQ, L);
1823 __ stop("super_check_offset inconsistent");
1824 __ bind(L);
1825 }
1826 #endif //ASSERT
1827
1828 gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized);
1829
1830 // save the original count
1831 __ mov(count_save, count);
1832
1833 // Copy from low to high addresses
1834 __ mov(start_to, to); // Save destination array start address
1835 __ b(L_load_element);
1836
1837 // ======== begin loop ========
1838 // (Loop is rotated; its entry is L_load_element.)
1839 // Loop control:
1840 // for (; count != 0; count--) {
1841 // copied_oop = load_heap_oop(from++);
1842 // ... generate_type_check ...;
1843 // store_heap_oop(to++, copied_oop);
1844 // }
1845 __ align(OptoLoopAlignment);
1846
1847 __ BIND(L_store_element);
1848 __ store_heap_oop(__ post(to, UseCompressedOops ? 4 : 8), copied_oop); // store the oop
|