1632 }
1633
1634 // !!!!! Special hack to get all types of calls to specify the byte offset
1635 // from the start of the call to the point where the return address
1636 // will point.
1637 int MachCallStaticJavaNode::ret_addr_offset() const
1638 {
1639 int offset = 5; // 5 bytes from start of call to where return address points
1640 offset += clear_avx_size();
1641 return offset;
1642 }
1643
1644 int MachCallDynamicJavaNode::ret_addr_offset() const
1645 {
1646 int offset = 15; // 15 bytes from start of call to where return address points
1647 offset += clear_avx_size();
1648 return offset;
1649 }
1650
1651 int MachCallRuntimeNode::ret_addr_offset() const {
1652 int offset = 13; // movq r10,#addr; callq (r10)
1653 if (this->ideal_Opcode() != Op_CallLeafVector) {
1654 offset += clear_avx_size();
1655 }
1656 return offset;
1657 }
1658 //
1659 // Compute padding required for nodes which need alignment
1660 //
1661
1662 // The address of the call instruction needs to be 4-byte aligned to
1663 // ensure that it does not span a cache line so that it can be patched.
1664 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
1665 {
1666 current_offset += clear_avx_size(); // skip vzeroupper
1667 current_offset += 1; // skip call opcode byte
1668 return align_up(current_offset, alignment_required()) - current_offset;
1669 }
1670
1671 // The address of the call instruction needs to be 4-byte aligned to
1870 st->print("\n\t");
1871 st->print("# stack alignment check");
1872 #endif
1873 }
1874 if (C->stub_function() != nullptr) {
1875 st->print("\n\t");
1876 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
1877 st->print("\n\t");
1878 st->print("je fast_entry\t");
1879 st->print("\n\t");
1880 st->print("call #nmethod_entry_barrier_stub\t");
1881 st->print("\n\tfast_entry:");
1882 }
1883 st->cr();
1884 }
1885 #endif
1886
1887 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1888 Compile* C = ra_->C;
1889
1890 int framesize = C->output()->frame_size_in_bytes();
1891 int bangsize = C->output()->bang_size_in_bytes();
1892
1893 if (C->clinit_barrier_on_entry()) {
1894 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
1895 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1896
1897 Label L_skip_barrier;
1898 Register klass = rscratch1;
1899
1900 __ mov_metadata(klass, C->method()->holder()->constant_encoding());
1901 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
1902
1903 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
1904
1905 __ bind(L_skip_barrier);
1906 }
1907
1908 __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
1909
1910 C->output()->set_frame_complete(__ offset());
1911
1912 if (C->has_mach_constant_base_node()) {
1913 // NOTE: We set the table base offset here because users might be
1914 // emitted before MachConstantBaseNode.
1915 ConstantTable& constant_table = C->output()->constant_table();
1916 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1917 }
1918 }
1919
1920 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1921 {
1922 return MachNode::size(ra_); // too many variables; just compute it
1923 // the hard way
1924 }
1925
1926 int MachPrologNode::reloc() const
1927 {
1928 return 0; // a large enough number
1929 }
1930
1931 //=============================================================================
1932 #ifndef PRODUCT
1933 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1934 {
1935 Compile* C = ra_->C;
1936 if (generate_vzeroupper(C)) {
1937 st->print("vzeroupper");
1938 st->cr(); st->print("\t");
1939 }
1940
1941 int framesize = C->output()->frame_size_in_bytes();
1942 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1943 // Remove word for return adr already pushed
1944 // and RBP
1952 st->print_cr("popq rbp");
1953 if (do_polling() && C->is_method_compilation()) {
1954 st->print("\t");
1955 st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t"
1956 "ja #safepoint_stub\t"
1957 "# Safepoint: poll for GC");
1958 }
1959 }
1960 #endif
1961
1962 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
1963 {
1964 Compile* C = ra_->C;
1965
1966 if (generate_vzeroupper(C)) {
1967 // Clear upper bits of YMM registers when current compiled code uses
1968 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1969 __ vzeroupper();
1970 }
1971
1972 int framesize = C->output()->frame_size_in_bytes();
1973 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1974 // Remove word for return adr already pushed
1975 // and RBP
1976 framesize -= 2*wordSize;
1977
1978 // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
1979
1980 if (framesize) {
1981 __ addq(rsp, framesize);
1982 }
1983
1984 __ popq(rbp);
1985
1986 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1987 __ reserved_stack_check();
1988 }
1989
1990 if (do_polling() && C->is_method_compilation()) {
1991 Label dummy_label;
1992 Label* code_stub = &dummy_label;
1993 if (!C->output()->in_scratch_emit_size()) {
1994 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1995 C->output()->add_stub(stub);
1996 code_stub = &stub->entry();
1997 }
1998 __ relocate(relocInfo::poll_return_type);
1999 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
2000 }
2001 }
2002
2003 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
2004 {
2005 return MachNode::size(ra_); // too many variables; just compute it
2006 // the hard way
2007 }
2008
2009 int MachEpilogNode::reloc() const
2010 {
2011 return 2; // a large enough number
2012 }
2013
2014 const Pipeline* MachEpilogNode::pipeline() const
2015 {
2016 return MachNode::pipeline_class();
2017 }
2018
2019 //=============================================================================
2020
2021 enum RC {
2022 rc_bad,
2023 rc_int,
2024 rc_kreg,
2025 rc_float,
2026 rc_stack
2027 };
2028
2588 #endif
2589
2590 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2591 {
2592 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2593 int reg = ra_->get_encode(this);
2594
2595 __ lea(as_Register(reg), Address(rsp, offset));
2596 }
2597
2598 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
2599 {
2600 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2601 if (ra_->get_encode(this) > 15) {
2602 return (offset < 0x80) ? 6 : 9; // REX2
2603 } else {
2604 return (offset < 0x80) ? 5 : 8; // REX
2605 }
2606 }
2607
2608 //=============================================================================
2609 #ifndef PRODUCT
2610 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2611 {
2612 st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2613 st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
2614 st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
2615 }
2616 #endif
2617
2618 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2619 {
2620 __ ic_check(InteriorEntryAlignment);
2621 }
2622
2623 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2624 {
2625 return MachNode::size(ra_); // too many variables; just compute it
2626 // the hard way
2627 }
2628
2629
2630 //=============================================================================
2631
2632 bool Matcher::supports_vector_calling_convention(void) {
2633 return EnableVectorSupport;
2634 }
2635
2636 static bool is_ndd_demotable_opr1(const MachNode* mdef) {
2637 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr1) != 0);
2638 }
2639
2640 static bool is_ndd_demotable_opr2(const MachNode* mdef) {
2641 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr2) != 0);
2642 }
2643
2644 #ifdef ASSERT
2645 static bool is_ndd_demotable(const MachNode* mdef) {
2646 return (is_ndd_demotable_opr1(mdef) || is_ndd_demotable_opr2(mdef));
2647 }
2648 #endif
4586 }
4587 __ post_call_nop();
4588 %}
4589
4590 enc_class Java_Dynamic_Call(method meth) %{
4591 __ ic_call((address)$meth$$method, resolved_method_index(masm));
4592 __ post_call_nop();
4593 %}
4594
4595 enc_class call_epilog %{
4596 if (VerifyStackAtCalls) {
4597 // Check that stack depth is unchanged: find majik cookie on stack
4598 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
4599 Label L;
4600 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
4601 __ jccb(Assembler::equal, L);
4602 // Die if stack mismatch
4603 __ int3();
4604 __ bind(L);
4605 }
4606 %}
4607
4608 %}
4609
4610 //----------FRAME--------------------------------------------------------------
4611 // Definition of frame structure and management information.
4612 //
4613 // S T A C K L A Y O U T Allocators stack-slot number
4614 // | (to get allocators register number
4615 // G Owned by | | v add OptoReg::stack0())
4616 // r CALLER | |
4617 // o | +--------+ pad to even-align allocators stack-slot
4618 // w V | pad0 | numbers; owned by CALLER
4619 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4620 // h ^ | in | 5
4621 // | | args | 4 Holes in incoming args owned by SELF
4622 // | | | | 3
4623 // | | +--------+
4624 // V | | old out| Empty on Intel, window on Sparc
4625 // | old |preserve| Must be even aligned.
5764 %}
5765 %}
5766
5767 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5768 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
5769 %{
5770 constraint(ALLOC_IN_RC(ptr_reg));
5771 predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5772 match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
5773
5774 op_cost(10);
5775 format %{"[$reg + $off + $idx << $scale]" %}
5776 interface(MEMORY_INTER) %{
5777 base($reg);
5778 index($idx);
5779 scale($scale);
5780 disp($off);
5781 %}
5782 %}
5783
5784 // Indirect Narrow Oop Plus Offset Operand
5785 // Note: x86 architecture doesn't support "scale * index + offset" without a base
5786 // we can't free r12 even with CompressedOops::base() == nullptr.
5787 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
5788 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5789 constraint(ALLOC_IN_RC(ptr_reg));
5790 match(AddP (DecodeN reg) off);
5791
5792 op_cost(10);
5793 format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
5794 interface(MEMORY_INTER) %{
5795 base(0xc); // R12
5796 index($reg);
5797 scale(0x3);
5798 disp($off);
5799 %}
5800 %}
5801
5802 // Indirect Memory Operand
5803 operand indirectNarrow(rRegN reg)
6273 %}
6274
6275 // Replaces legVec during post-selection cleanup. See above.
6276 operand legVecZ() %{
6277 constraint(ALLOC_IN_RC(vectorz_reg_legacy));
6278 match(VecZ);
6279
6280 format %{ %}
6281 interface(REG_INTER);
6282 %}
6283
6284 //----------OPERAND CLASSES----------------------------------------------------
6285 // Operand Classes are groups of operands that are used as to simplify
6286 // instruction definitions by not requiring the AD writer to specify separate
6287 // instructions for every form of operand when the instruction accepts
6288 // multiple operand types with the same basic encoding and format. The classic
6289 // case of this is memory operands.
6290
6291 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
6292 indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
6293 indCompressedOopOffset,
6294 indirectNarrow, indOffset8Narrow, indOffset32Narrow,
6295 indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
6296 indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
6297
6298 //----------PIPELINE-----------------------------------------------------------
6299 // Rules which define the behavior of the target architectures pipeline.
6300 pipeline %{
6301
6302 //----------ATTRIBUTES---------------------------------------------------------
6303 attributes %{
6304 variable_size_instructions; // Fixed size instructions
6305 max_instructions_per_bundle = 3; // Up to 3 instructions per bundle
6306 instruction_unit_size = 1; // An instruction is 1 bytes long
6307 instruction_fetch_unit_size = 16; // The processor fetches one line
6308 instruction_fetch_units = 1; // of 16 bytes
6309 %}
6310
6311 //----------RESOURCES----------------------------------------------------------
6312 // Resources are the functional units available to the machine
6313
8903 format %{ "MEMBAR-storestore (empty encoding)" %}
8904 ins_encode( );
8905 ins_pipe(empty);
8906 %}
8907
8908 //----------Move Instructions--------------------------------------------------
8909
8910 instruct castX2P(rRegP dst, rRegL src)
8911 %{
8912 match(Set dst (CastX2P src));
8913
8914 format %{ "movq $dst, $src\t# long->ptr" %}
8915 ins_encode %{
8916 if ($dst$$reg != $src$$reg) {
8917 __ movptr($dst$$Register, $src$$Register);
8918 }
8919 %}
8920 ins_pipe(ialu_reg_reg); // XXX
8921 %}
8922
8923 instruct castP2X(rRegL dst, rRegP src)
8924 %{
8925 match(Set dst (CastP2X src));
8926
8927 format %{ "movq $dst, $src\t# ptr -> long" %}
8928 ins_encode %{
8929 if ($dst$$reg != $src$$reg) {
8930 __ movptr($dst$$Register, $src$$Register);
8931 }
8932 %}
8933 ins_pipe(ialu_reg_reg); // XXX
8934 %}
8935
8936 // Convert oop into int for vectors alignment masking
8937 instruct convP2I(rRegI dst, rRegP src)
8938 %{
8939 match(Set dst (ConvL2I (CastP2X src)));
8940
8941 format %{ "movl $dst, $src\t# ptr -> int" %}
8942 ins_encode %{
14669 effect(DEF dst, USE src);
14670 ins_cost(100);
14671 format %{ "movd $dst,$src\t# MoveI2F" %}
14672 ins_encode %{
14673 __ movdl($dst$$XMMRegister, $src$$Register);
14674 %}
14675 ins_pipe( pipe_slow );
14676 %}
14677
14678 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
14679 match(Set dst (MoveL2D src));
14680 effect(DEF dst, USE src);
14681 ins_cost(100);
14682 format %{ "movd $dst,$src\t# MoveL2D" %}
14683 ins_encode %{
14684 __ movdq($dst$$XMMRegister, $src$$Register);
14685 %}
14686 ins_pipe( pipe_slow );
14687 %}
14688
14689 // Fast clearing of an array
14690 // Small non-constant lenght ClearArray for non-AVX512 targets.
14691 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
14692 Universe dummy, rFlagsReg cr)
14693 %{
14694 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
14695 match(Set dummy (ClearArray cnt base));
14696 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
14697
14698 format %{ $$template
14699 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14700 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14701 $$emit$$"jg LARGE\n\t"
14702 $$emit$$"dec rcx\n\t"
14703 $$emit$$"js DONE\t# Zero length\n\t"
14704 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14705 $$emit$$"dec rcx\n\t"
14706 $$emit$$"jge LOOP\n\t"
14707 $$emit$$"jmp DONE\n\t"
14708 $$emit$$"# LARGE:\n\t"
14709 if (UseFastStosb) {
14710 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14711 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14712 } else if (UseXMMForObjInit) {
14713 $$emit$$"mov rdi,rax\n\t"
14714 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14715 $$emit$$"jmpq L_zero_64_bytes\n\t"
14716 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14724 $$emit$$"jl L_tail\n\t"
14725 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14726 $$emit$$"add 0x20,rax\n\t"
14727 $$emit$$"sub 0x4,rcx\n\t"
14728 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14729 $$emit$$"add 0x4,rcx\n\t"
14730 $$emit$$"jle L_end\n\t"
14731 $$emit$$"dec rcx\n\t"
14732 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14733 $$emit$$"vmovq xmm0,(rax)\n\t"
14734 $$emit$$"add 0x8,rax\n\t"
14735 $$emit$$"dec rcx\n\t"
14736 $$emit$$"jge L_sloop\n\t"
14737 $$emit$$"# L_end:\n\t"
14738 } else {
14739 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14740 }
14741 $$emit$$"# DONE"
14742 %}
14743 ins_encode %{
14744 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14745 $tmp$$XMMRegister, false, knoreg);
14746 %}
14747 ins_pipe(pipe_slow);
14748 %}
14749
14750 // Small non-constant length ClearArray for AVX512 targets.
14751 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
14752 Universe dummy, rFlagsReg cr)
14753 %{
14754 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
14755 match(Set dummy (ClearArray cnt base));
14756 ins_cost(125);
14757 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
14758
14759 format %{ $$template
14760 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14761 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14762 $$emit$$"jg LARGE\n\t"
14763 $$emit$$"dec rcx\n\t"
14764 $$emit$$"js DONE\t# Zero length\n\t"
14765 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14766 $$emit$$"dec rcx\n\t"
14767 $$emit$$"jge LOOP\n\t"
14768 $$emit$$"jmp DONE\n\t"
14769 $$emit$$"# LARGE:\n\t"
14770 if (UseFastStosb) {
14771 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14772 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14773 } else if (UseXMMForObjInit) {
14774 $$emit$$"mov rdi,rax\n\t"
14775 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14776 $$emit$$"jmpq L_zero_64_bytes\n\t"
14777 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14785 $$emit$$"jl L_tail\n\t"
14786 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14787 $$emit$$"add 0x20,rax\n\t"
14788 $$emit$$"sub 0x4,rcx\n\t"
14789 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14790 $$emit$$"add 0x4,rcx\n\t"
14791 $$emit$$"jle L_end\n\t"
14792 $$emit$$"dec rcx\n\t"
14793 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14794 $$emit$$"vmovq xmm0,(rax)\n\t"
14795 $$emit$$"add 0x8,rax\n\t"
14796 $$emit$$"dec rcx\n\t"
14797 $$emit$$"jge L_sloop\n\t"
14798 $$emit$$"# L_end:\n\t"
14799 } else {
14800 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14801 }
14802 $$emit$$"# DONE"
14803 %}
14804 ins_encode %{
14805 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14806 $tmp$$XMMRegister, false, $ktmp$$KRegister);
14807 %}
14808 ins_pipe(pipe_slow);
14809 %}
14810
14811 // Large non-constant length ClearArray for non-AVX512 targets.
14812 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
14813 Universe dummy, rFlagsReg cr)
14814 %{
14815 predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
14816 match(Set dummy (ClearArray cnt base));
14817 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
14818
14819 format %{ $$template
14820 if (UseFastStosb) {
14821 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14822 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14823 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
14824 } else if (UseXMMForObjInit) {
14825 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
14826 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14827 $$emit$$"jmpq L_zero_64_bytes\n\t"
14828 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14829 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14830 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14831 $$emit$$"add 0x40,rax\n\t"
14832 $$emit$$"# L_zero_64_bytes:\n\t"
14833 $$emit$$"sub 0x8,rcx\n\t"
14834 $$emit$$"jge L_loop\n\t"
14835 $$emit$$"add 0x4,rcx\n\t"
14836 $$emit$$"jl L_tail\n\t"
14837 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14838 $$emit$$"add 0x20,rax\n\t"
14839 $$emit$$"sub 0x4,rcx\n\t"
14840 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14841 $$emit$$"add 0x4,rcx\n\t"
14842 $$emit$$"jle L_end\n\t"
14843 $$emit$$"dec rcx\n\t"
14844 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14845 $$emit$$"vmovq xmm0,(rax)\n\t"
14846 $$emit$$"add 0x8,rax\n\t"
14847 $$emit$$"dec rcx\n\t"
14848 $$emit$$"jge L_sloop\n\t"
14849 $$emit$$"# L_end:\n\t"
14850 } else {
14851 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14852 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
14853 }
14854 %}
14855 ins_encode %{
14856 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14857 $tmp$$XMMRegister, true, knoreg);
14858 %}
14859 ins_pipe(pipe_slow);
14860 %}
14861
14862 // Large non-constant length ClearArray for AVX512 targets.
14863 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
14864 Universe dummy, rFlagsReg cr)
14865 %{
14866 predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
14867 match(Set dummy (ClearArray cnt base));
14868 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
14869
14870 format %{ $$template
14871 if (UseFastStosb) {
14872 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14873 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14874 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
14875 } else if (UseXMMForObjInit) {
14876 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
14877 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14878 $$emit$$"jmpq L_zero_64_bytes\n\t"
14879 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14880 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14881 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14882 $$emit$$"add 0x40,rax\n\t"
14883 $$emit$$"# L_zero_64_bytes:\n\t"
14884 $$emit$$"sub 0x8,rcx\n\t"
14885 $$emit$$"jge L_loop\n\t"
14886 $$emit$$"add 0x4,rcx\n\t"
14887 $$emit$$"jl L_tail\n\t"
14888 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14889 $$emit$$"add 0x20,rax\n\t"
14890 $$emit$$"sub 0x4,rcx\n\t"
14891 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14892 $$emit$$"add 0x4,rcx\n\t"
14893 $$emit$$"jle L_end\n\t"
14894 $$emit$$"dec rcx\n\t"
14895 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14896 $$emit$$"vmovq xmm0,(rax)\n\t"
14897 $$emit$$"add 0x8,rax\n\t"
14898 $$emit$$"dec rcx\n\t"
14899 $$emit$$"jge L_sloop\n\t"
14900 $$emit$$"# L_end:\n\t"
14901 } else {
14902 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14903 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
14904 }
14905 %}
14906 ins_encode %{
14907 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14908 $tmp$$XMMRegister, true, $ktmp$$KRegister);
14909 %}
14910 ins_pipe(pipe_slow);
14911 %}
14912
14913 // Small constant length ClearArray for AVX512 targets.
14914 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
14915 %{
14916 predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
14917 match(Set dummy (ClearArray cnt base));
14918 ins_cost(100);
14919 effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
14920 format %{ "clear_mem_imm $base , $cnt \n\t" %}
14921 ins_encode %{
14922 __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
14923 %}
14924 ins_pipe(pipe_slow);
14925 %}
14926
14927 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14928 rax_RegI result, legRegD tmp1, rFlagsReg cr)
14929 %{
14930 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
14931 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14932 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14933
14934 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14935 ins_encode %{
14936 __ string_compare($str1$$Register, $str2$$Register,
14937 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14938 $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
14939 %}
14940 ins_pipe( pipe_slow );
14941 %}
14942
16780 effect(USE meth);
16781
16782 ins_cost(300);
16783 format %{ "call_leaf,runtime " %}
16784 ins_encode(clear_avx, Java_To_Runtime(meth));
16785 ins_pipe(pipe_slow);
16786 %}
16787
16788 // Call runtime without safepoint and with vector arguments
16789 instruct CallLeafDirectVector(method meth)
16790 %{
16791 match(CallLeafVector);
16792 effect(USE meth);
16793
16794 ins_cost(300);
16795 format %{ "call_leaf,vector " %}
16796 ins_encode(Java_To_Runtime(meth));
16797 ins_pipe(pipe_slow);
16798 %}
16799
16800 // Call runtime without safepoint
16801 instruct CallLeafNoFPDirect(method meth)
16802 %{
16803 match(CallLeafNoFP);
16804 effect(USE meth);
16805
16806 ins_cost(300);
16807 format %{ "call_leaf_nofp,runtime " %}
16808 ins_encode(clear_avx, Java_To_Runtime(meth));
16809 ins_pipe(pipe_slow);
16810 %}
16811
16812 // Return Instruction
16813 // Remove the return address & jump to it.
16814 // Notice: We always emit a nop after a ret to make sure there is room
16815 // for safepoint patching
16816 instruct Ret()
16817 %{
16818 match(Return);
16819
16820 format %{ "ret" %}
16821 ins_encode %{
16822 __ ret(0);
|
1632 }
1633
1634 // !!!!! Special hack to get all types of calls to specify the byte offset
1635 // from the start of the call to the point where the return address
1636 // will point.
1637 int MachCallStaticJavaNode::ret_addr_offset() const
1638 {
1639 int offset = 5; // 5 bytes from start of call to where return address points
1640 offset += clear_avx_size();
1641 return offset;
1642 }
1643
1644 int MachCallDynamicJavaNode::ret_addr_offset() const
1645 {
1646 int offset = 15; // 15 bytes from start of call to where return address points
1647 offset += clear_avx_size();
1648 return offset;
1649 }
1650
1651 int MachCallRuntimeNode::ret_addr_offset() const {
1652 if (_entry_point == nullptr) {
1653 // CallLeafNoFPInDirect
1654 return 3; // callq (register)
1655 }
1656 int offset = 13; // movq r10,#addr; callq (r10)
1657 if (this->ideal_Opcode() != Op_CallLeafVector) {
1658 offset += clear_avx_size();
1659 }
1660 return offset;
1661 }
1662 //
1663 // Compute padding required for nodes which need alignment
1664 //
1665
1666 // The address of the call instruction needs to be 4-byte aligned to
1667 // ensure that it does not span a cache line so that it can be patched.
1668 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
1669 {
1670 current_offset += clear_avx_size(); // skip vzeroupper
1671 current_offset += 1; // skip call opcode byte
1672 return align_up(current_offset, alignment_required()) - current_offset;
1673 }
1674
1675 // The address of the call instruction needs to be 4-byte aligned to
1874 st->print("\n\t");
1875 st->print("# stack alignment check");
1876 #endif
1877 }
1878 if (C->stub_function() != nullptr) {
1879 st->print("\n\t");
1880 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
1881 st->print("\n\t");
1882 st->print("je fast_entry\t");
1883 st->print("\n\t");
1884 st->print("call #nmethod_entry_barrier_stub\t");
1885 st->print("\n\tfast_entry:");
1886 }
1887 st->cr();
1888 }
1889 #endif
1890
1891 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1892 Compile* C = ra_->C;
1893
1894 __ verified_entry(C);
1895
1896 if (ra_->C->stub_function() == nullptr) {
1897 __ entry_barrier();
1898 }
1899
1900 if (!Compile::current()->output()->in_scratch_emit_size()) {
1901 __ bind(*_verified_entry);
1902 }
1903
1904 C->output()->set_frame_complete(__ offset());
1905
1906 if (C->has_mach_constant_base_node()) {
1907 // NOTE: We set the table base offset here because users might be
1908 // emitted before MachConstantBaseNode.
1909 ConstantTable& constant_table = C->output()->constant_table();
1910 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1911 }
1912 }
1913
1914
1915 int MachPrologNode::reloc() const
1916 {
1917 return 0; // a large enough number
1918 }
1919
1920 //=============================================================================
1921 #ifndef PRODUCT
1922 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1923 {
1924 Compile* C = ra_->C;
1925 if (generate_vzeroupper(C)) {
1926 st->print("vzeroupper");
1927 st->cr(); st->print("\t");
1928 }
1929
1930 int framesize = C->output()->frame_size_in_bytes();
1931 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1932 // Remove word for return adr already pushed
1933 // and RBP
1941 st->print_cr("popq rbp");
1942 if (do_polling() && C->is_method_compilation()) {
1943 st->print("\t");
1944 st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t"
1945 "ja #safepoint_stub\t"
1946 "# Safepoint: poll for GC");
1947 }
1948 }
1949 #endif
1950
1951 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
1952 {
1953 Compile* C = ra_->C;
1954
1955 if (generate_vzeroupper(C)) {
1956 // Clear upper bits of YMM registers when current compiled code uses
1957 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1958 __ vzeroupper();
1959 }
1960
1961 // Subtract two words to account for return address and rbp
1962 int initial_framesize = C->output()->frame_size_in_bytes() - 2*wordSize;
1963 __ remove_frame(initial_framesize, C->needs_stack_repair());
1964
1965 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1966 __ reserved_stack_check();
1967 }
1968
1969 if (do_polling() && C->is_method_compilation()) {
1970 Label dummy_label;
1971 Label* code_stub = &dummy_label;
1972 if (!C->output()->in_scratch_emit_size()) {
1973 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1974 C->output()->add_stub(stub);
1975 code_stub = &stub->entry();
1976 }
1977 __ relocate(relocInfo::poll_return_type);
1978 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1979 }
1980 }
1981
1982 int MachEpilogNode::reloc() const
1983 {
1984 return 2; // a large enough number
1985 }
1986
1987 const Pipeline* MachEpilogNode::pipeline() const
1988 {
1989 return MachNode::pipeline_class();
1990 }
1991
1992 //=============================================================================
1993
1994 enum RC {
1995 rc_bad,
1996 rc_int,
1997 rc_kreg,
1998 rc_float,
1999 rc_stack
2000 };
2001
2561 #endif
2562
2563 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2564 {
2565 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2566 int reg = ra_->get_encode(this);
2567
2568 __ lea(as_Register(reg), Address(rsp, offset));
2569 }
2570
2571 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
2572 {
2573 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2574 if (ra_->get_encode(this) > 15) {
2575 return (offset < 0x80) ? 6 : 9; // REX2
2576 } else {
2577 return (offset < 0x80) ? 5 : 8; // REX
2578 }
2579 }
2580
2581 //=============================================================================
2582 #ifndef PRODUCT
2583 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2584 {
2585 st->print_cr("MachVEPNode");
2586 }
2587 #endif
2588
2589 void MachVEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2590 {
2591 CodeBuffer* cbuf = masm->code();
2592 if (!_verified) {
2593 __ ic_check(1);
2594 } else {
2595 if (ra_->C->stub_function() == nullptr) {
2596 // Emit the entry barrier in a temporary frame before unpacking because
2597 // it can deopt, which would require packing the scalarized args again.
2598 __ verified_entry(ra_->C, 0);
2599 __ entry_barrier();
2600 int initial_framesize = ra_->C->output()->frame_size_in_bytes() - 2*wordSize;
2601 __ remove_frame(initial_framesize, false);
2602 }
2603 // Unpack inline type args passed as oop and then jump to
2604 // the verified entry point (skipping the unverified entry).
2605 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2606 // Emit code for verified entry and save increment for stack repair on return
2607 __ verified_entry(ra_->C, sp_inc);
2608 if (Compile::current()->output()->in_scratch_emit_size()) {
2609 Label dummy_verified_entry;
2610 __ jmp(dummy_verified_entry);
2611 } else {
2612 __ jmp(*_verified_entry);
2613 }
2614 }
2615 if (ra_->C->stub_function() == nullptr) {
2616 // Pad so that the next call to MachVEPNode::emit() starts out with the
2617 // correct alignment. This is needed by entry_barrier() to align the
2618 // compare. But unfortunately we need to align all 4 MachVEPNodes because
2619 // entry point offsets are computed using scratch_emit_size(), so starting
2620 // alignment must match the alignment of the scratch buffer, otherwise the sizes
2621 // will be off.
2622 __ align(4);
2623 }
2624 }
2625
2626 //=============================================================================
2627 #ifndef PRODUCT
2628 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2629 {
2630 st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2631 st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
2632 st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
2633 }
2634 #endif
2635
2636 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2637 {
2638 __ ic_check(InteriorEntryAlignment);
2639 }
2640
2641
2642 //=============================================================================
2643
2644 bool Matcher::supports_vector_calling_convention(void) {
2645 return EnableVectorSupport;
2646 }
2647
2648 static bool is_ndd_demotable_opr1(const MachNode* mdef) {
2649 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr1) != 0);
2650 }
2651
2652 static bool is_ndd_demotable_opr2(const MachNode* mdef) {
2653 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr2) != 0);
2654 }
2655
2656 #ifdef ASSERT
2657 static bool is_ndd_demotable(const MachNode* mdef) {
2658 return (is_ndd_demotable_opr1(mdef) || is_ndd_demotable_opr2(mdef));
2659 }
2660 #endif
4598 }
4599 __ post_call_nop();
4600 %}
4601
4602 enc_class Java_Dynamic_Call(method meth) %{
4603 __ ic_call((address)$meth$$method, resolved_method_index(masm));
4604 __ post_call_nop();
4605 %}
4606
4607 enc_class call_epilog %{
4608 if (VerifyStackAtCalls) {
4609 // Check that stack depth is unchanged: find majik cookie on stack
4610 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
4611 Label L;
4612 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
4613 __ jccb(Assembler::equal, L);
4614 // Die if stack mismatch
4615 __ int3();
4616 __ bind(L);
4617 }
4618 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
4619 // The last return value is not set by the callee but used to pass the null marker to compiled code.
4620 // Search for the corresponding projection, get the register and emit code that initializes it.
4621 uint con = (tf()->range_cc()->cnt() - 1);
4622 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
4623 ProjNode* proj = fast_out(i)->as_Proj();
4624 if (proj->_con == con) {
4625 // Set null marker if rax is non-null (a non-null value is returned buffered or scalarized)
4626 OptoReg::Name optoReg = ra_->get_reg_first(proj);
4627 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
4628 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
4629 __ testq(rax, rax);
4630 __ setb(Assembler::notZero, toReg);
4631 __ movzbl(toReg, toReg);
4632 if (reg->is_stack()) {
4633 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
4634 __ movq(Address(rsp, st_off), toReg);
4635 }
4636 break;
4637 }
4638 }
4639 if (return_value_is_used()) {
4640 // An inline type is returned as fields in multiple registers.
4641 // Rax either contains an oop if the inline type is buffered or a pointer
4642 // to the corresponding InlineKlass with the lowest bit set to 1. Zero rax
4643 // if the lowest bit is set to allow C2 to use the oop after null checking.
4644 // rax &= (rax & 1) - 1
4645 __ movptr(rscratch1, rax);
4646 __ andptr(rscratch1, 0x1);
4647 __ subptr(rscratch1, 0x1);
4648 __ andptr(rax, rscratch1);
4649 }
4650 }
4651 %}
4652
4653 %}
4654
4655 //----------FRAME--------------------------------------------------------------
4656 // Definition of frame structure and management information.
4657 //
4658 // S T A C K L A Y O U T Allocators stack-slot number
4659 // | (to get allocators register number
4660 // G Owned by | | v add OptoReg::stack0())
4661 // r CALLER | |
4662 // o | +--------+ pad to even-align allocators stack-slot
4663 // w V | pad0 | numbers; owned by CALLER
4664 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4665 // h ^ | in | 5
4666 // | | args | 4 Holes in incoming args owned by SELF
4667 // | | | | 3
4668 // | | +--------+
4669 // V | | old out| Empty on Intel, window on Sparc
4670 // | old |preserve| Must be even aligned.
5809 %}
5810 %}
5811
5812 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5813 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
5814 %{
5815 constraint(ALLOC_IN_RC(ptr_reg));
5816 predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5817 match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
5818
5819 op_cost(10);
5820 format %{"[$reg + $off + $idx << $scale]" %}
5821 interface(MEMORY_INTER) %{
5822 base($reg);
5823 index($idx);
5824 scale($scale);
5825 disp($off);
5826 %}
5827 %}
5828
5829 // Indirect Narrow Oop Operand
5830 operand indCompressedOop(rRegN reg) %{
5831 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5832 constraint(ALLOC_IN_RC(ptr_reg));
5833 match(DecodeN reg);
5834
5835 op_cost(10);
5836 format %{"[R12 + $reg << 3] (compressed oop addressing)" %}
5837 interface(MEMORY_INTER) %{
5838 base(0xc); // R12
5839 index($reg);
5840 scale(0x3);
5841 disp(0x0);
5842 %}
5843 %}
5844
5845 // Indirect Narrow Oop Plus Offset Operand
5846 // Note: x86 architecture doesn't support "scale * index + offset" without a base
5847 // we can't free r12 even with CompressedOops::base() == nullptr.
5848 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
5849 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5850 constraint(ALLOC_IN_RC(ptr_reg));
5851 match(AddP (DecodeN reg) off);
5852
5853 op_cost(10);
5854 format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
5855 interface(MEMORY_INTER) %{
5856 base(0xc); // R12
5857 index($reg);
5858 scale(0x3);
5859 disp($off);
5860 %}
5861 %}
5862
5863 // Indirect Memory Operand
5864 operand indirectNarrow(rRegN reg)
6334 %}
6335
6336 // Replaces legVec during post-selection cleanup. See above.
6337 operand legVecZ() %{
6338 constraint(ALLOC_IN_RC(vectorz_reg_legacy));
6339 match(VecZ);
6340
6341 format %{ %}
6342 interface(REG_INTER);
6343 %}
6344
6345 //----------OPERAND CLASSES----------------------------------------------------
6346 // Operand Classes are groups of operands that are used as to simplify
6347 // instruction definitions by not requiring the AD writer to specify separate
6348 // instructions for every form of operand when the instruction accepts
6349 // multiple operand types with the same basic encoding and format. The classic
6350 // case of this is memory operands.
6351
6352 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
6353 indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
6354 indCompressedOop, indCompressedOopOffset,
6355 indirectNarrow, indOffset8Narrow, indOffset32Narrow,
6356 indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
6357 indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
6358
6359 //----------PIPELINE-----------------------------------------------------------
6360 // Rules which define the behavior of the target architectures pipeline.
6361 pipeline %{
6362
6363 //----------ATTRIBUTES---------------------------------------------------------
6364 attributes %{
6365 variable_size_instructions; // Fixed size instructions
6366 max_instructions_per_bundle = 3; // Up to 3 instructions per bundle
6367 instruction_unit_size = 1; // An instruction is 1 bytes long
6368 instruction_fetch_unit_size = 16; // The processor fetches one line
6369 instruction_fetch_units = 1; // of 16 bytes
6370 %}
6371
6372 //----------RESOURCES----------------------------------------------------------
6373 // Resources are the functional units available to the machine
6374
8964 format %{ "MEMBAR-storestore (empty encoding)" %}
8965 ins_encode( );
8966 ins_pipe(empty);
8967 %}
8968
8969 //----------Move Instructions--------------------------------------------------
8970
8971 instruct castX2P(rRegP dst, rRegL src)
8972 %{
8973 match(Set dst (CastX2P src));
8974
8975 format %{ "movq $dst, $src\t# long->ptr" %}
8976 ins_encode %{
8977 if ($dst$$reg != $src$$reg) {
8978 __ movptr($dst$$Register, $src$$Register);
8979 }
8980 %}
8981 ins_pipe(ialu_reg_reg); // XXX
8982 %}
8983
8984 instruct castI2N(rRegN dst, rRegI src)
8985 %{
8986 match(Set dst (CastI2N src));
8987
8988 format %{ "movq $dst, $src\t# int -> narrow ptr" %}
8989 ins_encode %{
8990 if ($dst$$reg != $src$$reg) {
8991 __ movl($dst$$Register, $src$$Register);
8992 }
8993 %}
8994 ins_pipe(ialu_reg_reg); // XXX
8995 %}
8996
8997 instruct castN2X(rRegL dst, rRegN src)
8998 %{
8999 match(Set dst (CastP2X src));
9000
9001 format %{ "movq $dst, $src\t# ptr -> long" %}
9002 ins_encode %{
9003 if ($dst$$reg != $src$$reg) {
9004 __ movptr($dst$$Register, $src$$Register);
9005 }
9006 %}
9007 ins_pipe(ialu_reg_reg); // XXX
9008 %}
9009
9010 instruct castP2X(rRegL dst, rRegP src)
9011 %{
9012 match(Set dst (CastP2X src));
9013
9014 format %{ "movq $dst, $src\t# ptr -> long" %}
9015 ins_encode %{
9016 if ($dst$$reg != $src$$reg) {
9017 __ movptr($dst$$Register, $src$$Register);
9018 }
9019 %}
9020 ins_pipe(ialu_reg_reg); // XXX
9021 %}
9022
9023 // Convert oop into int for vectors alignment masking
9024 instruct convP2I(rRegI dst, rRegP src)
9025 %{
9026 match(Set dst (ConvL2I (CastP2X src)));
9027
9028 format %{ "movl $dst, $src\t# ptr -> int" %}
9029 ins_encode %{
14756 effect(DEF dst, USE src);
14757 ins_cost(100);
14758 format %{ "movd $dst,$src\t# MoveI2F" %}
14759 ins_encode %{
14760 __ movdl($dst$$XMMRegister, $src$$Register);
14761 %}
14762 ins_pipe( pipe_slow );
14763 %}
14764
14765 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
14766 match(Set dst (MoveL2D src));
14767 effect(DEF dst, USE src);
14768 ins_cost(100);
14769 format %{ "movd $dst,$src\t# MoveL2D" %}
14770 ins_encode %{
14771 __ movdq($dst$$XMMRegister, $src$$Register);
14772 %}
14773 ins_pipe( pipe_slow );
14774 %}
14775
14776
14777 // Fast clearing of an array
14778 // Small non-constant lenght ClearArray for non-AVX512 targets.
14779 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
14780 Universe dummy, rFlagsReg cr)
14781 %{
14782 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
14783 match(Set dummy (ClearArray (Binary cnt base) val));
14784 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
14785
14786 format %{ $$template
14787 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14788 $$emit$$"jg LARGE\n\t"
14789 $$emit$$"dec rcx\n\t"
14790 $$emit$$"js DONE\t# Zero length\n\t"
14791 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14792 $$emit$$"dec rcx\n\t"
14793 $$emit$$"jge LOOP\n\t"
14794 $$emit$$"jmp DONE\n\t"
14795 $$emit$$"# LARGE:\n\t"
14796 if (UseFastStosb) {
14797 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14798 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14799 } else if (UseXMMForObjInit) {
14800 $$emit$$"movdq $tmp, $val\n\t"
14801 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
14802 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
14803 $$emit$$"jmpq L_zero_64_bytes\n\t"
14804 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14805 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14806 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
14807 $$emit$$"add 0x40,rax\n\t"
14808 $$emit$$"# L_zero_64_bytes:\n\t"
14809 $$emit$$"sub 0x8,rcx\n\t"
14810 $$emit$$"jge L_loop\n\t"
14811 $$emit$$"add 0x4,rcx\n\t"
14812 $$emit$$"jl L_tail\n\t"
14813 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14814 $$emit$$"add 0x20,rax\n\t"
14815 $$emit$$"sub 0x4,rcx\n\t"
14816 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14817 $$emit$$"add 0x4,rcx\n\t"
14818 $$emit$$"jle L_end\n\t"
14819 $$emit$$"dec rcx\n\t"
14820 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14821 $$emit$$"vmovq xmm0,(rax)\n\t"
14822 $$emit$$"add 0x8,rax\n\t"
14823 $$emit$$"dec rcx\n\t"
14824 $$emit$$"jge L_sloop\n\t"
14825 $$emit$$"# L_end:\n\t"
14826 } else {
14827 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14828 }
14829 $$emit$$"# DONE"
14830 %}
14831 ins_encode %{
14832 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14833 $tmp$$XMMRegister, false, false);
14834 %}
14835 ins_pipe(pipe_slow);
14836 %}
14837
14838 instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
14839 Universe dummy, rFlagsReg cr)
14840 %{
14841 predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
14842 match(Set dummy (ClearArray (Binary cnt base) val));
14843 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
14844
14845 format %{ $$template
14846 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14847 $$emit$$"jg LARGE\n\t"
14848 $$emit$$"dec rcx\n\t"
14849 $$emit$$"js DONE\t# Zero length\n\t"
14850 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14851 $$emit$$"dec rcx\n\t"
14852 $$emit$$"jge LOOP\n\t"
14853 $$emit$$"jmp DONE\n\t"
14854 $$emit$$"# LARGE:\n\t"
14855 if (UseXMMForObjInit) {
14856 $$emit$$"movdq $tmp, $val\n\t"
14857 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
14858 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
14859 $$emit$$"jmpq L_zero_64_bytes\n\t"
14860 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14861 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14862 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
14863 $$emit$$"add 0x40,rax\n\t"
14864 $$emit$$"# L_zero_64_bytes:\n\t"
14865 $$emit$$"sub 0x8,rcx\n\t"
14866 $$emit$$"jge L_loop\n\t"
14867 $$emit$$"add 0x4,rcx\n\t"
14868 $$emit$$"jl L_tail\n\t"
14869 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14870 $$emit$$"add 0x20,rax\n\t"
14871 $$emit$$"sub 0x4,rcx\n\t"
14872 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14873 $$emit$$"add 0x4,rcx\n\t"
14874 $$emit$$"jle L_end\n\t"
14875 $$emit$$"dec rcx\n\t"
14876 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14877 $$emit$$"vmovq xmm0,(rax)\n\t"
14878 $$emit$$"add 0x8,rax\n\t"
14879 $$emit$$"dec rcx\n\t"
14880 $$emit$$"jge L_sloop\n\t"
14881 $$emit$$"# L_end:\n\t"
14882 } else {
14883 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14884 }
14885 $$emit$$"# DONE"
14886 %}
14887 ins_encode %{
14888 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14889 $tmp$$XMMRegister, false, true);
14890 %}
14891 ins_pipe(pipe_slow);
14892 %}
14893
14894 // Small non-constant length ClearArray for AVX512 targets.
14895 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
14896 Universe dummy, rFlagsReg cr)
14897 %{
14898 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
14899 match(Set dummy (ClearArray (Binary cnt base) val));
14900 ins_cost(125);
14901 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
14902
14903 format %{ $$template
14904 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14905 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14906 $$emit$$"jg LARGE\n\t"
14907 $$emit$$"dec rcx\n\t"
14908 $$emit$$"js DONE\t# Zero length\n\t"
14909 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14910 $$emit$$"dec rcx\n\t"
14911 $$emit$$"jge LOOP\n\t"
14912 $$emit$$"jmp DONE\n\t"
14913 $$emit$$"# LARGE:\n\t"
14914 if (UseFastStosb) {
14915 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14916 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14917 } else if (UseXMMForObjInit) {
14918 $$emit$$"mov rdi,rax\n\t"
14919 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14920 $$emit$$"jmpq L_zero_64_bytes\n\t"
14921 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14929 $$emit$$"jl L_tail\n\t"
14930 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14931 $$emit$$"add 0x20,rax\n\t"
14932 $$emit$$"sub 0x4,rcx\n\t"
14933 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14934 $$emit$$"add 0x4,rcx\n\t"
14935 $$emit$$"jle L_end\n\t"
14936 $$emit$$"dec rcx\n\t"
14937 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14938 $$emit$$"vmovq xmm0,(rax)\n\t"
14939 $$emit$$"add 0x8,rax\n\t"
14940 $$emit$$"dec rcx\n\t"
14941 $$emit$$"jge L_sloop\n\t"
14942 $$emit$$"# L_end:\n\t"
14943 } else {
14944 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14945 }
14946 $$emit$$"# DONE"
14947 %}
14948 ins_encode %{
14949 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14950 $tmp$$XMMRegister, false, false, $ktmp$$KRegister);
14951 %}
14952 ins_pipe(pipe_slow);
14953 %}
14954
14955 instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
14956 Universe dummy, rFlagsReg cr)
14957 %{
14958 predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
14959 match(Set dummy (ClearArray (Binary cnt base) val));
14960 ins_cost(125);
14961 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
14962
14963 format %{ $$template
14964 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14965 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14966 $$emit$$"jg LARGE\n\t"
14967 $$emit$$"dec rcx\n\t"
14968 $$emit$$"js DONE\t# Zero length\n\t"
14969 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14970 $$emit$$"dec rcx\n\t"
14971 $$emit$$"jge LOOP\n\t"
14972 $$emit$$"jmp DONE\n\t"
14973 $$emit$$"# LARGE:\n\t"
14974 if (UseFastStosb) {
14975 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14976 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14977 } else if (UseXMMForObjInit) {
14978 $$emit$$"mov rdi,rax\n\t"
14979 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14980 $$emit$$"jmpq L_zero_64_bytes\n\t"
14981 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14989 $$emit$$"jl L_tail\n\t"
14990 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14991 $$emit$$"add 0x20,rax\n\t"
14992 $$emit$$"sub 0x4,rcx\n\t"
14993 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14994 $$emit$$"add 0x4,rcx\n\t"
14995 $$emit$$"jle L_end\n\t"
14996 $$emit$$"dec rcx\n\t"
14997 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14998 $$emit$$"vmovq xmm0,(rax)\n\t"
14999 $$emit$$"add 0x8,rax\n\t"
15000 $$emit$$"dec rcx\n\t"
15001 $$emit$$"jge L_sloop\n\t"
15002 $$emit$$"# L_end:\n\t"
15003 } else {
15004 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
15005 }
15006 $$emit$$"# DONE"
15007 %}
15008 ins_encode %{
15009 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15010 $tmp$$XMMRegister, false, true, $ktmp$$KRegister);
15011 %}
15012 ins_pipe(pipe_slow);
15013 %}
15014
15015 // Large non-constant length ClearArray for non-AVX512 targets.
15016 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
15017 Universe dummy, rFlagsReg cr)
15018 %{
15019 predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
15020 match(Set dummy (ClearArray (Binary cnt base) val));
15021 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
15022
15023 format %{ $$template
15024 if (UseFastStosb) {
15025 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15026 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15027 } else if (UseXMMForObjInit) {
15028 $$emit$$"movdq $tmp, $val\n\t"
15029 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
15030 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
15031 $$emit$$"jmpq L_zero_64_bytes\n\t"
15032 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15033 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15034 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
15035 $$emit$$"add 0x40,rax\n\t"
15036 $$emit$$"# L_zero_64_bytes:\n\t"
15037 $$emit$$"sub 0x8,rcx\n\t"
15038 $$emit$$"jge L_loop\n\t"
15039 $$emit$$"add 0x4,rcx\n\t"
15040 $$emit$$"jl L_tail\n\t"
15041 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15042 $$emit$$"add 0x20,rax\n\t"
15043 $$emit$$"sub 0x4,rcx\n\t"
15044 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15045 $$emit$$"add 0x4,rcx\n\t"
15046 $$emit$$"jle L_end\n\t"
15047 $$emit$$"dec rcx\n\t"
15048 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15049 $$emit$$"vmovq xmm0,(rax)\n\t"
15050 $$emit$$"add 0x8,rax\n\t"
15051 $$emit$$"dec rcx\n\t"
15052 $$emit$$"jge L_sloop\n\t"
15053 $$emit$$"# L_end:\n\t"
15054 } else {
15055 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15056 }
15057 %}
15058 ins_encode %{
15059 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15060 $tmp$$XMMRegister, true, false);
15061 %}
15062 ins_pipe(pipe_slow);
15063 %}
15064
15065 instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
15066 Universe dummy, rFlagsReg cr)
15067 %{
15068 predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
15069 match(Set dummy (ClearArray (Binary cnt base) val));
15070 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
15071
15072 format %{ $$template
15073 if (UseXMMForObjInit) {
15074 $$emit$$"movdq $tmp, $val\n\t"
15075 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
15076 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
15077 $$emit$$"jmpq L_zero_64_bytes\n\t"
15078 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15079 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15080 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
15081 $$emit$$"add 0x40,rax\n\t"
15082 $$emit$$"# L_zero_64_bytes:\n\t"
15083 $$emit$$"sub 0x8,rcx\n\t"
15084 $$emit$$"jge L_loop\n\t"
15085 $$emit$$"add 0x4,rcx\n\t"
15086 $$emit$$"jl L_tail\n\t"
15087 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15088 $$emit$$"add 0x20,rax\n\t"
15089 $$emit$$"sub 0x4,rcx\n\t"
15090 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15091 $$emit$$"add 0x4,rcx\n\t"
15092 $$emit$$"jle L_end\n\t"
15093 $$emit$$"dec rcx\n\t"
15094 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15095 $$emit$$"vmovq xmm0,(rax)\n\t"
15096 $$emit$$"add 0x8,rax\n\t"
15097 $$emit$$"dec rcx\n\t"
15098 $$emit$$"jge L_sloop\n\t"
15099 $$emit$$"# L_end:\n\t"
15100 } else {
15101 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15102 }
15103 %}
15104 ins_encode %{
15105 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15106 $tmp$$XMMRegister, true, true);
15107 %}
15108 ins_pipe(pipe_slow);
15109 %}
15110
15111 // Large non-constant length ClearArray for AVX512 targets.
15112 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
15113 Universe dummy, rFlagsReg cr)
15114 %{
15115 predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
15116 match(Set dummy (ClearArray (Binary cnt base) val));
15117 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
15118
15119 format %{ $$template
15120 if (UseFastStosb) {
15121 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15122 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15123 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15124 } else if (UseXMMForObjInit) {
15125 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
15126 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
15127 $$emit$$"jmpq L_zero_64_bytes\n\t"
15128 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15129 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15130 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
15131 $$emit$$"add 0x40,rax\n\t"
15132 $$emit$$"# L_zero_64_bytes:\n\t"
15133 $$emit$$"sub 0x8,rcx\n\t"
15134 $$emit$$"jge L_loop\n\t"
15135 $$emit$$"add 0x4,rcx\n\t"
15136 $$emit$$"jl L_tail\n\t"
15137 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15138 $$emit$$"add 0x20,rax\n\t"
15139 $$emit$$"sub 0x4,rcx\n\t"
15140 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15141 $$emit$$"add 0x4,rcx\n\t"
15142 $$emit$$"jle L_end\n\t"
15143 $$emit$$"dec rcx\n\t"
15144 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15145 $$emit$$"vmovq xmm0,(rax)\n\t"
15146 $$emit$$"add 0x8,rax\n\t"
15147 $$emit$$"dec rcx\n\t"
15148 $$emit$$"jge L_sloop\n\t"
15149 $$emit$$"# L_end:\n\t"
15150 } else {
15151 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15152 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15153 }
15154 %}
15155 ins_encode %{
15156 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15157 $tmp$$XMMRegister, true, false, $ktmp$$KRegister);
15158 %}
15159 ins_pipe(pipe_slow);
15160 %}
15161
15162 instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
15163 Universe dummy, rFlagsReg cr)
15164 %{
15165 predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
15166 match(Set dummy (ClearArray (Binary cnt base) val));
15167 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
15168
15169 format %{ $$template
15170 if (UseFastStosb) {
15171 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15172 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15173 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15174 } else if (UseXMMForObjInit) {
15175 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
15176 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
15177 $$emit$$"jmpq L_zero_64_bytes\n\t"
15178 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15179 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15180 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
15181 $$emit$$"add 0x40,rax\n\t"
15182 $$emit$$"# L_zero_64_bytes:\n\t"
15183 $$emit$$"sub 0x8,rcx\n\t"
15184 $$emit$$"jge L_loop\n\t"
15185 $$emit$$"add 0x4,rcx\n\t"
15186 $$emit$$"jl L_tail\n\t"
15187 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15188 $$emit$$"add 0x20,rax\n\t"
15189 $$emit$$"sub 0x4,rcx\n\t"
15190 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15191 $$emit$$"add 0x4,rcx\n\t"
15192 $$emit$$"jle L_end\n\t"
15193 $$emit$$"dec rcx\n\t"
15194 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15195 $$emit$$"vmovq xmm0,(rax)\n\t"
15196 $$emit$$"add 0x8,rax\n\t"
15197 $$emit$$"dec rcx\n\t"
15198 $$emit$$"jge L_sloop\n\t"
15199 $$emit$$"# L_end:\n\t"
15200 } else {
15201 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15202 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15203 }
15204 %}
15205 ins_encode %{
15206 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15207 $tmp$$XMMRegister, true, true, $ktmp$$KRegister);
15208 %}
15209 ins_pipe(pipe_slow);
15210 %}
15211
15212 // Small constant length ClearArray for AVX512 targets.
15213 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rax_RegL val, kReg ktmp, Universe dummy, rFlagsReg cr)
15214 %{
15215 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
15216 ((MaxVectorSize >= 32) && VM_Version::supports_avx512vl()));
15217 match(Set dummy (ClearArray (Binary cnt base) val));
15218 ins_cost(100);
15219 effect(TEMP tmp, USE_KILL val, TEMP ktmp, KILL cr);
15220 format %{ "clear_mem_imm $base , $cnt \n\t" %}
15221 ins_encode %{
15222 __ clear_mem($base$$Register, $cnt$$constant, $val$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
15223 %}
15224 ins_pipe(pipe_slow);
15225 %}
15226
15227 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15228 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15229 %{
15230 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
15231 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15232 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15233
15234 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15235 ins_encode %{
15236 __ string_compare($str1$$Register, $str2$$Register,
15237 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15238 $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
15239 %}
15240 ins_pipe( pipe_slow );
15241 %}
15242
17080 effect(USE meth);
17081
17082 ins_cost(300);
17083 format %{ "call_leaf,runtime " %}
17084 ins_encode(clear_avx, Java_To_Runtime(meth));
17085 ins_pipe(pipe_slow);
17086 %}
17087
17088 // Call runtime without safepoint and with vector arguments
17089 instruct CallLeafDirectVector(method meth)
17090 %{
17091 match(CallLeafVector);
17092 effect(USE meth);
17093
17094 ins_cost(300);
17095 format %{ "call_leaf,vector " %}
17096 ins_encode(Java_To_Runtime(meth));
17097 ins_pipe(pipe_slow);
17098 %}
17099
17100 // Call runtime without safepoint
17101 // entry point is null, target holds the address to call
17102 instruct CallLeafNoFPInDirect(rRegP target)
17103 %{
17104 predicate(n->as_Call()->entry_point() == nullptr);
17105 match(CallLeafNoFP target);
17106
17107 ins_cost(300);
17108 format %{ "call_leaf_nofp,runtime indirect " %}
17109 ins_encode %{
17110 __ call($target$$Register);
17111 %}
17112
17113 ins_pipe(pipe_slow);
17114 %}
17115
17116 // Call runtime without safepoint
17117 instruct CallLeafNoFPDirect(method meth)
17118 %{
17119 predicate(n->as_Call()->entry_point() != nullptr);
17120 match(CallLeafNoFP);
17121 effect(USE meth);
17122
17123 ins_cost(300);
17124 format %{ "call_leaf_nofp,runtime " %}
17125 ins_encode(clear_avx, Java_To_Runtime(meth));
17126 ins_pipe(pipe_slow);
17127 %}
17128
17129 // Return Instruction
17130 // Remove the return address & jump to it.
17131 // Notice: We always emit a nop after a ret to make sure there is room
17132 // for safepoint patching
17133 instruct Ret()
17134 %{
17135 match(Return);
17136
17137 format %{ "ret" %}
17138 ins_encode %{
17139 __ ret(0);
|