1632 }
1633
1634 // !!!!! Special hack to get all types of calls to specify the byte offset
1635 // from the start of the call to the point where the return address
1636 // will point.
1637 int MachCallStaticJavaNode::ret_addr_offset()
1638 {
1639 int offset = 5; // 5 bytes from start of call to where return address points
1640 offset += clear_avx_size();
1641 return offset;
1642 }
1643
1644 int MachCallDynamicJavaNode::ret_addr_offset()
1645 {
1646 int offset = 15; // 15 bytes from start of call to where return address points
1647 offset += clear_avx_size();
1648 return offset;
1649 }
1650
1651 int MachCallRuntimeNode::ret_addr_offset() {
1652 int offset = 13; // movq r10,#addr; callq (r10)
1653 if (this->ideal_Opcode() != Op_CallLeafVector) {
1654 offset += clear_avx_size();
1655 }
1656 return offset;
1657 }
1658 //
1659 // Compute padding required for nodes which need alignment
1660 //
1661
1662 // The address of the call instruction needs to be 4-byte aligned to
1663 // ensure that it does not span a cache line so that it can be patched.
1664 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
1665 {
1666 current_offset += clear_avx_size(); // skip vzeroupper
1667 current_offset += 1; // skip call opcode byte
1668 return align_up(current_offset, alignment_required()) - current_offset;
1669 }
1670
1671 // The address of the call instruction needs to be 4-byte aligned to
1870 st->print("\n\t");
1871 st->print("# stack alignment check");
1872 #endif
1873 }
1874 if (C->stub_function() != nullptr) {
1875 st->print("\n\t");
1876 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
1877 st->print("\n\t");
1878 st->print("je fast_entry\t");
1879 st->print("\n\t");
1880 st->print("call #nmethod_entry_barrier_stub\t");
1881 st->print("\n\tfast_entry:");
1882 }
1883 st->cr();
1884 }
1885 #endif
1886
1887 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1888 Compile* C = ra_->C;
1889
1890 int framesize = C->output()->frame_size_in_bytes();
1891 int bangsize = C->output()->bang_size_in_bytes();
1892
1893 if (C->clinit_barrier_on_entry()) {
1894 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
1895 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1896
1897 Label L_skip_barrier;
1898 Register klass = rscratch1;
1899
1900 __ mov_metadata(klass, C->method()->holder()->constant_encoding());
1901 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
1902
1903 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
1904
1905 __ bind(L_skip_barrier);
1906 }
1907
1908 __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
1909
1910 C->output()->set_frame_complete(__ offset());
1911
1912 if (C->has_mach_constant_base_node()) {
1913 // NOTE: We set the table base offset here because users might be
1914 // emitted before MachConstantBaseNode.
1915 ConstantTable& constant_table = C->output()->constant_table();
1916 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1917 }
1918 }
1919
1920 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1921 {
1922 return MachNode::size(ra_); // too many variables; just compute it
1923 // the hard way
1924 }
1925
1926 int MachPrologNode::reloc() const
1927 {
1928 return 0; // a large enough number
1929 }
1930
1931 //=============================================================================
1932 #ifndef PRODUCT
1933 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1934 {
1935 Compile* C = ra_->C;
1936 if (generate_vzeroupper(C)) {
1937 st->print("vzeroupper");
1938 st->cr(); st->print("\t");
1939 }
1940
1941 int framesize = C->output()->frame_size_in_bytes();
1942 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1943 // Remove word for return adr already pushed
1944 // and RBP
1952 st->print_cr("popq rbp");
1953 if (do_polling() && C->is_method_compilation()) {
1954 st->print("\t");
1955 st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t"
1956 "ja #safepoint_stub\t"
1957 "# Safepoint: poll for GC");
1958 }
1959 }
1960 #endif
1961
1962 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
1963 {
1964 Compile* C = ra_->C;
1965
1966 if (generate_vzeroupper(C)) {
1967 // Clear upper bits of YMM registers when current compiled code uses
1968 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1969 __ vzeroupper();
1970 }
1971
1972 int framesize = C->output()->frame_size_in_bytes();
1973 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1974 // Remove word for return adr already pushed
1975 // and RBP
1976 framesize -= 2*wordSize;
1977
1978 // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
1979
1980 if (framesize) {
1981 __ addq(rsp, framesize);
1982 }
1983
1984 __ popq(rbp);
1985
1986 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1987 __ reserved_stack_check();
1988 }
1989
1990 if (do_polling() && C->is_method_compilation()) {
1991 Label dummy_label;
1992 Label* code_stub = &dummy_label;
1993 if (!C->output()->in_scratch_emit_size()) {
1994 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1995 C->output()->add_stub(stub);
1996 code_stub = &stub->entry();
1997 }
1998 __ relocate(relocInfo::poll_return_type);
1999 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
2000 }
2001 }
2002
2003 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
2004 {
2005 return MachNode::size(ra_); // too many variables; just compute it
2006 // the hard way
2007 }
2008
2009 int MachEpilogNode::reloc() const
2010 {
2011 return 2; // a large enough number
2012 }
2013
2014 const Pipeline* MachEpilogNode::pipeline() const
2015 {
2016 return MachNode::pipeline_class();
2017 }
2018
2019 //=============================================================================
2020
2021 enum RC {
2022 rc_bad,
2023 rc_int,
2024 rc_kreg,
2025 rc_float,
2026 rc_stack
2027 };
2028
2590 #endif
2591
2592 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2593 {
2594 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2595 int reg = ra_->get_encode(this);
2596
2597 __ lea(as_Register(reg), Address(rsp, offset));
2598 }
2599
2600 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
2601 {
2602 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2603 if (ra_->get_encode(this) > 15) {
2604 return (offset < 0x80) ? 6 : 9; // REX2
2605 } else {
2606 return (offset < 0x80) ? 5 : 8; // REX
2607 }
2608 }
2609
2610 //=============================================================================
2611 #ifndef PRODUCT
2612 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2613 {
2614 st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2615 st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
2616 st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
2617 }
2618 #endif
2619
2620 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2621 {
2622 __ ic_check(InteriorEntryAlignment);
2623 }
2624
2625 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2626 {
2627 return MachNode::size(ra_); // too many variables; just compute it
2628 // the hard way
2629 }
2630
2631
2632 //=============================================================================
2633
2634 bool Matcher::supports_vector_calling_convention(void) {
2635 return EnableVectorSupport;
2636 }
2637
2638 static bool is_ndd_demotable_opr1(const MachNode* mdef) {
2639 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr1) != 0);
2640 }
2641
2642 static bool is_ndd_demotable_opr2(const MachNode* mdef) {
2643 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr2) != 0);
2644 }
2645
2646 #ifdef ASSERT
2647 static bool is_ndd_demotable(const MachNode* mdef) {
2648 return (is_ndd_demotable_opr1(mdef) || is_ndd_demotable_opr2(mdef));
2649 }
2650 #endif
4591 }
4592 __ post_call_nop();
4593 %}
4594
4595 enc_class Java_Dynamic_Call(method meth) %{
4596 __ ic_call((address)$meth$$method, resolved_method_index(masm));
4597 __ post_call_nop();
4598 %}
4599
4600 enc_class call_epilog %{
4601 if (VerifyStackAtCalls) {
4602 // Check that stack depth is unchanged: find majik cookie on stack
4603 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
4604 Label L;
4605 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
4606 __ jccb(Assembler::equal, L);
4607 // Die if stack mismatch
4608 __ int3();
4609 __ bind(L);
4610 }
4611 %}
4612
4613 %}
4614
4615 //----------FRAME--------------------------------------------------------------
4616 // Definition of frame structure and management information.
4617 //
4618 // S T A C K L A Y O U T Allocators stack-slot number
4619 // | (to get allocators register number
4620 // G Owned by | | v add OptoReg::stack0())
4621 // r CALLER | |
4622 // o | +--------+ pad to even-align allocators stack-slot
4623 // w V | pad0 | numbers; owned by CALLER
4624 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4625 // h ^ | in | 5
4626 // | | args | 4 Holes in incoming args owned by SELF
4627 // | | | | 3
4628 // | | +--------+
4629 // V | | old out| Empty on Intel, window on Sparc
4630 // | old |preserve| Must be even aligned.
5769 %}
5770 %}
5771
5772 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5773 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
5774 %{
5775 constraint(ALLOC_IN_RC(ptr_reg));
5776 predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5777 match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
5778
5779 op_cost(10);
5780 format %{"[$reg + $off + $idx << $scale]" %}
5781 interface(MEMORY_INTER) %{
5782 base($reg);
5783 index($idx);
5784 scale($scale);
5785 disp($off);
5786 %}
5787 %}
5788
5789 // Indirect Narrow Oop Plus Offset Operand
5790 // Note: x86 architecture doesn't support "scale * index + offset" without a base
5791 // we can't free r12 even with CompressedOops::base() == nullptr.
5792 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
5793 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5794 constraint(ALLOC_IN_RC(ptr_reg));
5795 match(AddP (DecodeN reg) off);
5796
5797 op_cost(10);
5798 format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
5799 interface(MEMORY_INTER) %{
5800 base(0xc); // R12
5801 index($reg);
5802 scale(0x3);
5803 disp($off);
5804 %}
5805 %}
5806
5807 // Indirect Memory Operand
5808 operand indirectNarrow(rRegN reg)
6278 %}
6279
6280 // Replaces legVec during post-selection cleanup. See above.
6281 operand legVecZ() %{
6282 constraint(ALLOC_IN_RC(vectorz_reg_legacy));
6283 match(VecZ);
6284
6285 format %{ %}
6286 interface(REG_INTER);
6287 %}
6288
6289 //----------OPERAND CLASSES----------------------------------------------------
6290 // Operand Classes are groups of operands that are used as to simplify
6291 // instruction definitions by not requiring the AD writer to specify separate
6292 // instructions for every form of operand when the instruction accepts
6293 // multiple operand types with the same basic encoding and format. The classic
6294 // case of this is memory operands.
6295
6296 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
6297 indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
6298 indCompressedOopOffset,
6299 indirectNarrow, indOffset8Narrow, indOffset32Narrow,
6300 indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
6301 indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
6302
6303 //----------PIPELINE-----------------------------------------------------------
6304 // Rules which define the behavior of the target architectures pipeline.
6305 pipeline %{
6306
6307 //----------ATTRIBUTES---------------------------------------------------------
6308 attributes %{
6309 variable_size_instructions; // Fixed size instructions
6310 max_instructions_per_bundle = 3; // Up to 3 instructions per bundle
6311 instruction_unit_size = 1; // An instruction is 1 bytes long
6312 instruction_fetch_unit_size = 16; // The processor fetches one line
6313 instruction_fetch_units = 1; // of 16 bytes
6314 %}
6315
6316 //----------RESOURCES----------------------------------------------------------
6317 // Resources are the functional units available to the machine
6318
8908 format %{ "MEMBAR-storestore (empty encoding)" %}
8909 ins_encode( );
8910 ins_pipe(empty);
8911 %}
8912
8913 //----------Move Instructions--------------------------------------------------
8914
8915 instruct castX2P(rRegP dst, rRegL src)
8916 %{
8917 match(Set dst (CastX2P src));
8918
8919 format %{ "movq $dst, $src\t# long->ptr" %}
8920 ins_encode %{
8921 if ($dst$$reg != $src$$reg) {
8922 __ movptr($dst$$Register, $src$$Register);
8923 }
8924 %}
8925 ins_pipe(ialu_reg_reg); // XXX
8926 %}
8927
8928 instruct castP2X(rRegL dst, rRegP src)
8929 %{
8930 match(Set dst (CastP2X src));
8931
8932 format %{ "movq $dst, $src\t# ptr -> long" %}
8933 ins_encode %{
8934 if ($dst$$reg != $src$$reg) {
8935 __ movptr($dst$$Register, $src$$Register);
8936 }
8937 %}
8938 ins_pipe(ialu_reg_reg); // XXX
8939 %}
8940
8941 // Convert oop into int for vectors alignment masking
8942 instruct convP2I(rRegI dst, rRegP src)
8943 %{
8944 match(Set dst (ConvL2I (CastP2X src)));
8945
8946 format %{ "movl $dst, $src\t# ptr -> int" %}
8947 ins_encode %{
14674 effect(DEF dst, USE src);
14675 ins_cost(100);
14676 format %{ "movd $dst,$src\t# MoveI2F" %}
14677 ins_encode %{
14678 __ movdl($dst$$XMMRegister, $src$$Register);
14679 %}
14680 ins_pipe( pipe_slow );
14681 %}
14682
14683 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
14684 match(Set dst (MoveL2D src));
14685 effect(DEF dst, USE src);
14686 ins_cost(100);
14687 format %{ "movd $dst,$src\t# MoveL2D" %}
14688 ins_encode %{
14689 __ movdq($dst$$XMMRegister, $src$$Register);
14690 %}
14691 ins_pipe( pipe_slow );
14692 %}
14693
14694 // Fast clearing of an array
14695 // Small non-constant lenght ClearArray for non-AVX512 targets.
14696 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
14697 Universe dummy, rFlagsReg cr)
14698 %{
14699 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
14700 match(Set dummy (ClearArray cnt base));
14701 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
14702
14703 format %{ $$template
14704 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14705 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14706 $$emit$$"jg LARGE\n\t"
14707 $$emit$$"dec rcx\n\t"
14708 $$emit$$"js DONE\t# Zero length\n\t"
14709 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14710 $$emit$$"dec rcx\n\t"
14711 $$emit$$"jge LOOP\n\t"
14712 $$emit$$"jmp DONE\n\t"
14713 $$emit$$"# LARGE:\n\t"
14714 if (UseFastStosb) {
14715 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14716 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14717 } else if (UseXMMForObjInit) {
14718 $$emit$$"mov rdi,rax\n\t"
14719 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14720 $$emit$$"jmpq L_zero_64_bytes\n\t"
14721 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14729 $$emit$$"jl L_tail\n\t"
14730 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14731 $$emit$$"add 0x20,rax\n\t"
14732 $$emit$$"sub 0x4,rcx\n\t"
14733 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14734 $$emit$$"add 0x4,rcx\n\t"
14735 $$emit$$"jle L_end\n\t"
14736 $$emit$$"dec rcx\n\t"
14737 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14738 $$emit$$"vmovq xmm0,(rax)\n\t"
14739 $$emit$$"add 0x8,rax\n\t"
14740 $$emit$$"dec rcx\n\t"
14741 $$emit$$"jge L_sloop\n\t"
14742 $$emit$$"# L_end:\n\t"
14743 } else {
14744 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14745 }
14746 $$emit$$"# DONE"
14747 %}
14748 ins_encode %{
14749 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14750 $tmp$$XMMRegister, false, knoreg);
14751 %}
14752 ins_pipe(pipe_slow);
14753 %}
14754
14755 // Small non-constant length ClearArray for AVX512 targets.
14756 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
14757 Universe dummy, rFlagsReg cr)
14758 %{
14759 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
14760 match(Set dummy (ClearArray cnt base));
14761 ins_cost(125);
14762 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
14763
14764 format %{ $$template
14765 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14766 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14767 $$emit$$"jg LARGE\n\t"
14768 $$emit$$"dec rcx\n\t"
14769 $$emit$$"js DONE\t# Zero length\n\t"
14770 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14771 $$emit$$"dec rcx\n\t"
14772 $$emit$$"jge LOOP\n\t"
14773 $$emit$$"jmp DONE\n\t"
14774 $$emit$$"# LARGE:\n\t"
14775 if (UseFastStosb) {
14776 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14777 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14778 } else if (UseXMMForObjInit) {
14779 $$emit$$"mov rdi,rax\n\t"
14780 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14781 $$emit$$"jmpq L_zero_64_bytes\n\t"
14782 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14790 $$emit$$"jl L_tail\n\t"
14791 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14792 $$emit$$"add 0x20,rax\n\t"
14793 $$emit$$"sub 0x4,rcx\n\t"
14794 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14795 $$emit$$"add 0x4,rcx\n\t"
14796 $$emit$$"jle L_end\n\t"
14797 $$emit$$"dec rcx\n\t"
14798 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14799 $$emit$$"vmovq xmm0,(rax)\n\t"
14800 $$emit$$"add 0x8,rax\n\t"
14801 $$emit$$"dec rcx\n\t"
14802 $$emit$$"jge L_sloop\n\t"
14803 $$emit$$"# L_end:\n\t"
14804 } else {
14805 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14806 }
14807 $$emit$$"# DONE"
14808 %}
14809 ins_encode %{
14810 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14811 $tmp$$XMMRegister, false, $ktmp$$KRegister);
14812 %}
14813 ins_pipe(pipe_slow);
14814 %}
14815
14816 // Large non-constant length ClearArray for non-AVX512 targets.
14817 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
14818 Universe dummy, rFlagsReg cr)
14819 %{
14820 predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
14821 match(Set dummy (ClearArray cnt base));
14822 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
14823
14824 format %{ $$template
14825 if (UseFastStosb) {
14826 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14827 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14828 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
14829 } else if (UseXMMForObjInit) {
14830 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
14831 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14832 $$emit$$"jmpq L_zero_64_bytes\n\t"
14833 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14834 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14835 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14836 $$emit$$"add 0x40,rax\n\t"
14837 $$emit$$"# L_zero_64_bytes:\n\t"
14838 $$emit$$"sub 0x8,rcx\n\t"
14839 $$emit$$"jge L_loop\n\t"
14840 $$emit$$"add 0x4,rcx\n\t"
14841 $$emit$$"jl L_tail\n\t"
14842 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14843 $$emit$$"add 0x20,rax\n\t"
14844 $$emit$$"sub 0x4,rcx\n\t"
14845 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14846 $$emit$$"add 0x4,rcx\n\t"
14847 $$emit$$"jle L_end\n\t"
14848 $$emit$$"dec rcx\n\t"
14849 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14850 $$emit$$"vmovq xmm0,(rax)\n\t"
14851 $$emit$$"add 0x8,rax\n\t"
14852 $$emit$$"dec rcx\n\t"
14853 $$emit$$"jge L_sloop\n\t"
14854 $$emit$$"# L_end:\n\t"
14855 } else {
14856 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14857 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
14858 }
14859 %}
14860 ins_encode %{
14861 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14862 $tmp$$XMMRegister, true, knoreg);
14863 %}
14864 ins_pipe(pipe_slow);
14865 %}
14866
14867 // Large non-constant length ClearArray for AVX512 targets.
14868 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
14869 Universe dummy, rFlagsReg cr)
14870 %{
14871 predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
14872 match(Set dummy (ClearArray cnt base));
14873 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
14874
14875 format %{ $$template
14876 if (UseFastStosb) {
14877 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14878 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14879 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
14880 } else if (UseXMMForObjInit) {
14881 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
14882 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14883 $$emit$$"jmpq L_zero_64_bytes\n\t"
14884 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14885 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14886 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14887 $$emit$$"add 0x40,rax\n\t"
14888 $$emit$$"# L_zero_64_bytes:\n\t"
14889 $$emit$$"sub 0x8,rcx\n\t"
14890 $$emit$$"jge L_loop\n\t"
14891 $$emit$$"add 0x4,rcx\n\t"
14892 $$emit$$"jl L_tail\n\t"
14893 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14894 $$emit$$"add 0x20,rax\n\t"
14895 $$emit$$"sub 0x4,rcx\n\t"
14896 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14897 $$emit$$"add 0x4,rcx\n\t"
14898 $$emit$$"jle L_end\n\t"
14899 $$emit$$"dec rcx\n\t"
14900 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14901 $$emit$$"vmovq xmm0,(rax)\n\t"
14902 $$emit$$"add 0x8,rax\n\t"
14903 $$emit$$"dec rcx\n\t"
14904 $$emit$$"jge L_sloop\n\t"
14905 $$emit$$"# L_end:\n\t"
14906 } else {
14907 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14908 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
14909 }
14910 %}
14911 ins_encode %{
14912 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14913 $tmp$$XMMRegister, true, $ktmp$$KRegister);
14914 %}
14915 ins_pipe(pipe_slow);
14916 %}
14917
14918 // Small constant length ClearArray for AVX512 targets.
14919 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
14920 %{
14921 predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
14922 match(Set dummy (ClearArray cnt base));
14923 ins_cost(100);
14924 effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
14925 format %{ "clear_mem_imm $base , $cnt \n\t" %}
14926 ins_encode %{
14927 __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
14928 %}
14929 ins_pipe(pipe_slow);
14930 %}
14931
14932 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14933 rax_RegI result, legRegD tmp1, rFlagsReg cr)
14934 %{
14935 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
14936 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14937 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14938
14939 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14940 ins_encode %{
14941 __ string_compare($str1$$Register, $str2$$Register,
14942 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14943 $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
14944 %}
14945 ins_pipe( pipe_slow );
14946 %}
14947
16785 effect(USE meth);
16786
16787 ins_cost(300);
16788 format %{ "call_leaf,runtime " %}
16789 ins_encode(clear_avx, Java_To_Runtime(meth));
16790 ins_pipe(pipe_slow);
16791 %}
16792
16793 // Call runtime without safepoint and with vector arguments
16794 instruct CallLeafDirectVector(method meth)
16795 %{
16796 match(CallLeafVector);
16797 effect(USE meth);
16798
16799 ins_cost(300);
16800 format %{ "call_leaf,vector " %}
16801 ins_encode(Java_To_Runtime(meth));
16802 ins_pipe(pipe_slow);
16803 %}
16804
16805 // Call runtime without safepoint
16806 instruct CallLeafNoFPDirect(method meth)
16807 %{
16808 match(CallLeafNoFP);
16809 effect(USE meth);
16810
16811 ins_cost(300);
16812 format %{ "call_leaf_nofp,runtime " %}
16813 ins_encode(clear_avx, Java_To_Runtime(meth));
16814 ins_pipe(pipe_slow);
16815 %}
16816
16817 // Return Instruction
16818 // Remove the return address & jump to it.
16819 // Notice: We always emit a nop after a ret to make sure there is room
16820 // for safepoint patching
16821 instruct Ret()
16822 %{
16823 match(Return);
16824
16825 format %{ "ret" %}
16826 ins_encode %{
16827 __ ret(0);
|
1632 }
1633
1634 // !!!!! Special hack to get all types of calls to specify the byte offset
1635 // from the start of the call to the point where the return address
1636 // will point.
1637 int MachCallStaticJavaNode::ret_addr_offset()
1638 {
1639 int offset = 5; // 5 bytes from start of call to where return address points
1640 offset += clear_avx_size();
1641 return offset;
1642 }
1643
1644 int MachCallDynamicJavaNode::ret_addr_offset()
1645 {
1646 int offset = 15; // 15 bytes from start of call to where return address points
1647 offset += clear_avx_size();
1648 return offset;
1649 }
1650
1651 int MachCallRuntimeNode::ret_addr_offset() {
1652 if (_entry_point == nullptr) {
1653 // CallLeafNoFPInDirect
1654 return 3; // callq (register)
1655 }
1656 int offset = 13; // movq r10,#addr; callq (r10)
1657 if (this->ideal_Opcode() != Op_CallLeafVector) {
1658 offset += clear_avx_size();
1659 }
1660 return offset;
1661 }
1662 //
1663 // Compute padding required for nodes which need alignment
1664 //
1665
1666 // The address of the call instruction needs to be 4-byte aligned to
1667 // ensure that it does not span a cache line so that it can be patched.
1668 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
1669 {
1670 current_offset += clear_avx_size(); // skip vzeroupper
1671 current_offset += 1; // skip call opcode byte
1672 return align_up(current_offset, alignment_required()) - current_offset;
1673 }
1674
1675 // The address of the call instruction needs to be 4-byte aligned to
1874 st->print("\n\t");
1875 st->print("# stack alignment check");
1876 #endif
1877 }
1878 if (C->stub_function() != nullptr) {
1879 st->print("\n\t");
1880 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
1881 st->print("\n\t");
1882 st->print("je fast_entry\t");
1883 st->print("\n\t");
1884 st->print("call #nmethod_entry_barrier_stub\t");
1885 st->print("\n\tfast_entry:");
1886 }
1887 st->cr();
1888 }
1889 #endif
1890
1891 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1892 Compile* C = ra_->C;
1893
1894 __ verified_entry(C);
1895
1896 if (ra_->C->stub_function() == nullptr) {
1897 __ entry_barrier();
1898 }
1899
1900 if (!Compile::current()->output()->in_scratch_emit_size()) {
1901 __ bind(*_verified_entry);
1902 }
1903
1904 C->output()->set_frame_complete(__ offset());
1905
1906 if (C->has_mach_constant_base_node()) {
1907 // NOTE: We set the table base offset here because users might be
1908 // emitted before MachConstantBaseNode.
1909 ConstantTable& constant_table = C->output()->constant_table();
1910 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1911 }
1912 }
1913
1914
1915 int MachPrologNode::reloc() const
1916 {
1917 return 0; // a large enough number
1918 }
1919
1920 //=============================================================================
1921 #ifndef PRODUCT
1922 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1923 {
1924 Compile* C = ra_->C;
1925 if (generate_vzeroupper(C)) {
1926 st->print("vzeroupper");
1927 st->cr(); st->print("\t");
1928 }
1929
1930 int framesize = C->output()->frame_size_in_bytes();
1931 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1932 // Remove word for return adr already pushed
1933 // and RBP
1941 st->print_cr("popq rbp");
1942 if (do_polling() && C->is_method_compilation()) {
1943 st->print("\t");
1944 st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t"
1945 "ja #safepoint_stub\t"
1946 "# Safepoint: poll for GC");
1947 }
1948 }
1949 #endif
1950
1951 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
1952 {
1953 Compile* C = ra_->C;
1954
1955 if (generate_vzeroupper(C)) {
1956 // Clear upper bits of YMM registers when current compiled code uses
1957 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1958 __ vzeroupper();
1959 }
1960
1961 // Subtract two words to account for return address and rbp
1962 int initial_framesize = C->output()->frame_size_in_bytes() - 2*wordSize;
1963 __ remove_frame(initial_framesize, C->needs_stack_repair());
1964
1965 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1966 __ reserved_stack_check();
1967 }
1968
1969 if (do_polling() && C->is_method_compilation()) {
1970 Label dummy_label;
1971 Label* code_stub = &dummy_label;
1972 if (!C->output()->in_scratch_emit_size()) {
1973 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1974 C->output()->add_stub(stub);
1975 code_stub = &stub->entry();
1976 }
1977 __ relocate(relocInfo::poll_return_type);
1978 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1979 }
1980 }
1981
1982 int MachEpilogNode::reloc() const
1983 {
1984 return 2; // a large enough number
1985 }
1986
1987 const Pipeline* MachEpilogNode::pipeline() const
1988 {
1989 return MachNode::pipeline_class();
1990 }
1991
1992 //=============================================================================
1993
1994 enum RC {
1995 rc_bad,
1996 rc_int,
1997 rc_kreg,
1998 rc_float,
1999 rc_stack
2000 };
2001
2563 #endif
2564
2565 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2566 {
2567 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2568 int reg = ra_->get_encode(this);
2569
2570 __ lea(as_Register(reg), Address(rsp, offset));
2571 }
2572
2573 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
2574 {
2575 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2576 if (ra_->get_encode(this) > 15) {
2577 return (offset < 0x80) ? 6 : 9; // REX2
2578 } else {
2579 return (offset < 0x80) ? 5 : 8; // REX
2580 }
2581 }
2582
2583 //=============================================================================
2584 #ifndef PRODUCT
2585 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2586 {
2587 st->print_cr("MachVEPNode");
2588 }
2589 #endif
2590
2591 void MachVEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2592 {
2593 CodeBuffer* cbuf = masm->code();
2594 if (!_verified) {
2595 __ ic_check(1);
2596 } else {
2597 if (ra_->C->stub_function() == nullptr) {
2598 // Emit the entry barrier in a temporary frame before unpacking because
2599 // it can deopt, which would require packing the scalarized args again.
2600 __ verified_entry(ra_->C, 0);
2601 __ entry_barrier();
2602 int initial_framesize = ra_->C->output()->frame_size_in_bytes() - 2*wordSize;
2603 __ remove_frame(initial_framesize, false);
2604 }
2605 // Unpack inline type args passed as oop and then jump to
2606 // the verified entry point (skipping the unverified entry).
2607 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2608 // Emit code for verified entry and save increment for stack repair on return
2609 __ verified_entry(ra_->C, sp_inc);
2610 if (Compile::current()->output()->in_scratch_emit_size()) {
2611 Label dummy_verified_entry;
2612 __ jmp(dummy_verified_entry);
2613 } else {
2614 __ jmp(*_verified_entry);
2615 }
2616 }
2617 if (ra_->C->stub_function() == nullptr) {
2618 // Pad so that the next call to MachVEPNode::emit() starts out with the
2619 // correct alignment. This is needed by entry_barrier() to align the
2620 // compare. But unfortunately we need to align all 4 MachVEPNodes because
2621 // entry point offsets are computed using scratch_emit_size(), so starting
2622 // alignment must match the alignment of the scratch buffer, otherwise the sizes
2623 // will be off.
2624 __ align(4);
2625 }
2626 }
2627
2628 //=============================================================================
2629 #ifndef PRODUCT
2630 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2631 {
2632 st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2633 st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
2634 st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
2635 }
2636 #endif
2637
2638 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2639 {
2640 __ ic_check(InteriorEntryAlignment);
2641 }
2642
2643
2644 //=============================================================================
2645
2646 bool Matcher::supports_vector_calling_convention(void) {
2647 return EnableVectorSupport;
2648 }
2649
2650 static bool is_ndd_demotable_opr1(const MachNode* mdef) {
2651 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr1) != 0);
2652 }
2653
2654 static bool is_ndd_demotable_opr2(const MachNode* mdef) {
2655 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr2) != 0);
2656 }
2657
2658 #ifdef ASSERT
2659 static bool is_ndd_demotable(const MachNode* mdef) {
2660 return (is_ndd_demotable_opr1(mdef) || is_ndd_demotable_opr2(mdef));
2661 }
2662 #endif
4603 }
4604 __ post_call_nop();
4605 %}
4606
4607 enc_class Java_Dynamic_Call(method meth) %{
4608 __ ic_call((address)$meth$$method, resolved_method_index(masm));
4609 __ post_call_nop();
4610 %}
4611
4612 enc_class call_epilog %{
4613 if (VerifyStackAtCalls) {
4614 // Check that stack depth is unchanged: find majik cookie on stack
4615 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
4616 Label L;
4617 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
4618 __ jccb(Assembler::equal, L);
4619 // Die if stack mismatch
4620 __ int3();
4621 __ bind(L);
4622 }
4623 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
4624 // The last return value is not set by the callee but used to pass the null marker to compiled code.
4625 // Search for the corresponding projection, get the register and emit code that initializes it.
4626 uint con = (tf()->range_cc()->cnt() - 1);
4627 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
4628 ProjNode* proj = fast_out(i)->as_Proj();
4629 if (proj->_con == con) {
4630 // Set null marker if rax is non-null (a non-null value is returned buffered or scalarized)
4631 OptoReg::Name optoReg = ra_->get_reg_first(proj);
4632 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
4633 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
4634 __ testq(rax, rax);
4635 __ setb(Assembler::notZero, toReg);
4636 __ movzbl(toReg, toReg);
4637 if (reg->is_stack()) {
4638 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
4639 __ movq(Address(rsp, st_off), toReg);
4640 }
4641 break;
4642 }
4643 }
4644 if (return_value_is_used()) {
4645 // An inline type is returned as fields in multiple registers.
4646 // Rax either contains an oop if the inline type is buffered or a pointer
4647 // to the corresponding InlineKlass with the lowest bit set to 1. Zero rax
4648 // if the lowest bit is set to allow C2 to use the oop after null checking.
4649 // rax &= (rax & 1) - 1
4650 __ movptr(rscratch1, rax);
4651 __ andptr(rscratch1, 0x1);
4652 __ subptr(rscratch1, 0x1);
4653 __ andptr(rax, rscratch1);
4654 }
4655 }
4656 %}
4657
4658 %}
4659
4660 //----------FRAME--------------------------------------------------------------
4661 // Definition of frame structure and management information.
4662 //
4663 // S T A C K L A Y O U T Allocators stack-slot number
4664 // | (to get allocators register number
4665 // G Owned by | | v add OptoReg::stack0())
4666 // r CALLER | |
4667 // o | +--------+ pad to even-align allocators stack-slot
4668 // w V | pad0 | numbers; owned by CALLER
4669 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4670 // h ^ | in | 5
4671 // | | args | 4 Holes in incoming args owned by SELF
4672 // | | | | 3
4673 // | | +--------+
4674 // V | | old out| Empty on Intel, window on Sparc
4675 // | old |preserve| Must be even aligned.
5814 %}
5815 %}
5816
5817 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5818 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
5819 %{
5820 constraint(ALLOC_IN_RC(ptr_reg));
5821 predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5822 match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
5823
5824 op_cost(10);
5825 format %{"[$reg + $off + $idx << $scale]" %}
5826 interface(MEMORY_INTER) %{
5827 base($reg);
5828 index($idx);
5829 scale($scale);
5830 disp($off);
5831 %}
5832 %}
5833
5834 // Indirect Narrow Oop Operand
5835 operand indCompressedOop(rRegN reg) %{
5836 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5837 constraint(ALLOC_IN_RC(ptr_reg));
5838 match(DecodeN reg);
5839
5840 op_cost(10);
5841 format %{"[R12 + $reg << 3] (compressed oop addressing)" %}
5842 interface(MEMORY_INTER) %{
5843 base(0xc); // R12
5844 index($reg);
5845 scale(0x3);
5846 disp(0x0);
5847 %}
5848 %}
5849
5850 // Indirect Narrow Oop Plus Offset Operand
5851 // Note: x86 architecture doesn't support "scale * index + offset" without a base
5852 // we can't free r12 even with CompressedOops::base() == nullptr.
5853 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
5854 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5855 constraint(ALLOC_IN_RC(ptr_reg));
5856 match(AddP (DecodeN reg) off);
5857
5858 op_cost(10);
5859 format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
5860 interface(MEMORY_INTER) %{
5861 base(0xc); // R12
5862 index($reg);
5863 scale(0x3);
5864 disp($off);
5865 %}
5866 %}
5867
5868 // Indirect Memory Operand
5869 operand indirectNarrow(rRegN reg)
6339 %}
6340
6341 // Replaces legVec during post-selection cleanup. See above.
6342 operand legVecZ() %{
6343 constraint(ALLOC_IN_RC(vectorz_reg_legacy));
6344 match(VecZ);
6345
6346 format %{ %}
6347 interface(REG_INTER);
6348 %}
6349
6350 //----------OPERAND CLASSES----------------------------------------------------
6351 // Operand Classes are groups of operands that are used as to simplify
6352 // instruction definitions by not requiring the AD writer to specify separate
6353 // instructions for every form of operand when the instruction accepts
6354 // multiple operand types with the same basic encoding and format. The classic
6355 // case of this is memory operands.
6356
6357 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
6358 indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
6359 indCompressedOop, indCompressedOopOffset,
6360 indirectNarrow, indOffset8Narrow, indOffset32Narrow,
6361 indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
6362 indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
6363
6364 //----------PIPELINE-----------------------------------------------------------
6365 // Rules which define the behavior of the target architectures pipeline.
6366 pipeline %{
6367
6368 //----------ATTRIBUTES---------------------------------------------------------
6369 attributes %{
6370 variable_size_instructions; // Fixed size instructions
6371 max_instructions_per_bundle = 3; // Up to 3 instructions per bundle
6372 instruction_unit_size = 1; // An instruction is 1 bytes long
6373 instruction_fetch_unit_size = 16; // The processor fetches one line
6374 instruction_fetch_units = 1; // of 16 bytes
6375 %}
6376
6377 //----------RESOURCES----------------------------------------------------------
6378 // Resources are the functional units available to the machine
6379
8969 format %{ "MEMBAR-storestore (empty encoding)" %}
8970 ins_encode( );
8971 ins_pipe(empty);
8972 %}
8973
8974 //----------Move Instructions--------------------------------------------------
8975
8976 instruct castX2P(rRegP dst, rRegL src)
8977 %{
8978 match(Set dst (CastX2P src));
8979
8980 format %{ "movq $dst, $src\t# long->ptr" %}
8981 ins_encode %{
8982 if ($dst$$reg != $src$$reg) {
8983 __ movptr($dst$$Register, $src$$Register);
8984 }
8985 %}
8986 ins_pipe(ialu_reg_reg); // XXX
8987 %}
8988
8989 instruct castI2N(rRegN dst, rRegI src)
8990 %{
8991 match(Set dst (CastI2N src));
8992
8993 format %{ "movq $dst, $src\t# int -> narrow ptr" %}
8994 ins_encode %{
8995 if ($dst$$reg != $src$$reg) {
8996 __ movl($dst$$Register, $src$$Register);
8997 }
8998 %}
8999 ins_pipe(ialu_reg_reg); // XXX
9000 %}
9001
9002 instruct castN2X(rRegL dst, rRegN src)
9003 %{
9004 match(Set dst (CastP2X src));
9005
9006 format %{ "movq $dst, $src\t# ptr -> long" %}
9007 ins_encode %{
9008 if ($dst$$reg != $src$$reg) {
9009 __ movptr($dst$$Register, $src$$Register);
9010 }
9011 %}
9012 ins_pipe(ialu_reg_reg); // XXX
9013 %}
9014
9015 instruct castP2X(rRegL dst, rRegP src)
9016 %{
9017 match(Set dst (CastP2X src));
9018
9019 format %{ "movq $dst, $src\t# ptr -> long" %}
9020 ins_encode %{
9021 if ($dst$$reg != $src$$reg) {
9022 __ movptr($dst$$Register, $src$$Register);
9023 }
9024 %}
9025 ins_pipe(ialu_reg_reg); // XXX
9026 %}
9027
9028 // Convert oop into int for vectors alignment masking
9029 instruct convP2I(rRegI dst, rRegP src)
9030 %{
9031 match(Set dst (ConvL2I (CastP2X src)));
9032
9033 format %{ "movl $dst, $src\t# ptr -> int" %}
9034 ins_encode %{
14761 effect(DEF dst, USE src);
14762 ins_cost(100);
14763 format %{ "movd $dst,$src\t# MoveI2F" %}
14764 ins_encode %{
14765 __ movdl($dst$$XMMRegister, $src$$Register);
14766 %}
14767 ins_pipe( pipe_slow );
14768 %}
14769
14770 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
14771 match(Set dst (MoveL2D src));
14772 effect(DEF dst, USE src);
14773 ins_cost(100);
14774 format %{ "movd $dst,$src\t# MoveL2D" %}
14775 ins_encode %{
14776 __ movdq($dst$$XMMRegister, $src$$Register);
14777 %}
14778 ins_pipe( pipe_slow );
14779 %}
14780
14781
14782 // Fast clearing of an array
14783 // Small non-constant lenght ClearArray for non-AVX512 targets.
14784 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
14785 Universe dummy, rFlagsReg cr)
14786 %{
14787 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
14788 match(Set dummy (ClearArray (Binary cnt base) val));
14789 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
14790
14791 format %{ $$template
14792 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14793 $$emit$$"jg LARGE\n\t"
14794 $$emit$$"dec rcx\n\t"
14795 $$emit$$"js DONE\t# Zero length\n\t"
14796 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14797 $$emit$$"dec rcx\n\t"
14798 $$emit$$"jge LOOP\n\t"
14799 $$emit$$"jmp DONE\n\t"
14800 $$emit$$"# LARGE:\n\t"
14801 if (UseFastStosb) {
14802 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14803 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14804 } else if (UseXMMForObjInit) {
14805 $$emit$$"movdq $tmp, $val\n\t"
14806 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
14807 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
14808 $$emit$$"jmpq L_zero_64_bytes\n\t"
14809 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14810 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14811 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
14812 $$emit$$"add 0x40,rax\n\t"
14813 $$emit$$"# L_zero_64_bytes:\n\t"
14814 $$emit$$"sub 0x8,rcx\n\t"
14815 $$emit$$"jge L_loop\n\t"
14816 $$emit$$"add 0x4,rcx\n\t"
14817 $$emit$$"jl L_tail\n\t"
14818 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14819 $$emit$$"add 0x20,rax\n\t"
14820 $$emit$$"sub 0x4,rcx\n\t"
14821 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14822 $$emit$$"add 0x4,rcx\n\t"
14823 $$emit$$"jle L_end\n\t"
14824 $$emit$$"dec rcx\n\t"
14825 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14826 $$emit$$"vmovq xmm0,(rax)\n\t"
14827 $$emit$$"add 0x8,rax\n\t"
14828 $$emit$$"dec rcx\n\t"
14829 $$emit$$"jge L_sloop\n\t"
14830 $$emit$$"# L_end:\n\t"
14831 } else {
14832 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14833 }
14834 $$emit$$"# DONE"
14835 %}
14836 ins_encode %{
14837 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14838 $tmp$$XMMRegister, false, false);
14839 %}
14840 ins_pipe(pipe_slow);
14841 %}
14842
14843 instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
14844 Universe dummy, rFlagsReg cr)
14845 %{
14846 predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
14847 match(Set dummy (ClearArray (Binary cnt base) val));
14848 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
14849
14850 format %{ $$template
14851 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14852 $$emit$$"jg LARGE\n\t"
14853 $$emit$$"dec rcx\n\t"
14854 $$emit$$"js DONE\t# Zero length\n\t"
14855 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14856 $$emit$$"dec rcx\n\t"
14857 $$emit$$"jge LOOP\n\t"
14858 $$emit$$"jmp DONE\n\t"
14859 $$emit$$"# LARGE:\n\t"
14860 if (UseXMMForObjInit) {
14861 $$emit$$"movdq $tmp, $val\n\t"
14862 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
14863 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
14864 $$emit$$"jmpq L_zero_64_bytes\n\t"
14865 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14866 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14867 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
14868 $$emit$$"add 0x40,rax\n\t"
14869 $$emit$$"# L_zero_64_bytes:\n\t"
14870 $$emit$$"sub 0x8,rcx\n\t"
14871 $$emit$$"jge L_loop\n\t"
14872 $$emit$$"add 0x4,rcx\n\t"
14873 $$emit$$"jl L_tail\n\t"
14874 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14875 $$emit$$"add 0x20,rax\n\t"
14876 $$emit$$"sub 0x4,rcx\n\t"
14877 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14878 $$emit$$"add 0x4,rcx\n\t"
14879 $$emit$$"jle L_end\n\t"
14880 $$emit$$"dec rcx\n\t"
14881 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14882 $$emit$$"vmovq xmm0,(rax)\n\t"
14883 $$emit$$"add 0x8,rax\n\t"
14884 $$emit$$"dec rcx\n\t"
14885 $$emit$$"jge L_sloop\n\t"
14886 $$emit$$"# L_end:\n\t"
14887 } else {
14888 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14889 }
14890 $$emit$$"# DONE"
14891 %}
14892 ins_encode %{
14893 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14894 $tmp$$XMMRegister, false, true);
14895 %}
14896 ins_pipe(pipe_slow);
14897 %}
14898
14899 // Small non-constant length ClearArray for AVX512 targets.
14900 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
14901 Universe dummy, rFlagsReg cr)
14902 %{
14903 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
14904 match(Set dummy (ClearArray (Binary cnt base) val));
14905 ins_cost(125);
14906 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
14907
14908 format %{ $$template
14909 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14910 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14911 $$emit$$"jg LARGE\n\t"
14912 $$emit$$"dec rcx\n\t"
14913 $$emit$$"js DONE\t# Zero length\n\t"
14914 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14915 $$emit$$"dec rcx\n\t"
14916 $$emit$$"jge LOOP\n\t"
14917 $$emit$$"jmp DONE\n\t"
14918 $$emit$$"# LARGE:\n\t"
14919 if (UseFastStosb) {
14920 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14921 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14922 } else if (UseXMMForObjInit) {
14923 $$emit$$"mov rdi,rax\n\t"
14924 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14925 $$emit$$"jmpq L_zero_64_bytes\n\t"
14926 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14934 $$emit$$"jl L_tail\n\t"
14935 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14936 $$emit$$"add 0x20,rax\n\t"
14937 $$emit$$"sub 0x4,rcx\n\t"
14938 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14939 $$emit$$"add 0x4,rcx\n\t"
14940 $$emit$$"jle L_end\n\t"
14941 $$emit$$"dec rcx\n\t"
14942 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14943 $$emit$$"vmovq xmm0,(rax)\n\t"
14944 $$emit$$"add 0x8,rax\n\t"
14945 $$emit$$"dec rcx\n\t"
14946 $$emit$$"jge L_sloop\n\t"
14947 $$emit$$"# L_end:\n\t"
14948 } else {
14949 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14950 }
14951 $$emit$$"# DONE"
14952 %}
14953 ins_encode %{
14954 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14955 $tmp$$XMMRegister, false, false, $ktmp$$KRegister);
14956 %}
14957 ins_pipe(pipe_slow);
14958 %}
14959
14960 instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
14961 Universe dummy, rFlagsReg cr)
14962 %{
14963 predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
14964 match(Set dummy (ClearArray (Binary cnt base) val));
14965 ins_cost(125);
14966 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
14967
14968 format %{ $$template
14969 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14970 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14971 $$emit$$"jg LARGE\n\t"
14972 $$emit$$"dec rcx\n\t"
14973 $$emit$$"js DONE\t# Zero length\n\t"
14974 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14975 $$emit$$"dec rcx\n\t"
14976 $$emit$$"jge LOOP\n\t"
14977 $$emit$$"jmp DONE\n\t"
14978 $$emit$$"# LARGE:\n\t"
14979 if (UseFastStosb) {
14980 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14981 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14982 } else if (UseXMMForObjInit) {
14983 $$emit$$"mov rdi,rax\n\t"
14984 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14985 $$emit$$"jmpq L_zero_64_bytes\n\t"
14986 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14994 $$emit$$"jl L_tail\n\t"
14995 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14996 $$emit$$"add 0x20,rax\n\t"
14997 $$emit$$"sub 0x4,rcx\n\t"
14998 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14999 $$emit$$"add 0x4,rcx\n\t"
15000 $$emit$$"jle L_end\n\t"
15001 $$emit$$"dec rcx\n\t"
15002 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15003 $$emit$$"vmovq xmm0,(rax)\n\t"
15004 $$emit$$"add 0x8,rax\n\t"
15005 $$emit$$"dec rcx\n\t"
15006 $$emit$$"jge L_sloop\n\t"
15007 $$emit$$"# L_end:\n\t"
15008 } else {
15009 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
15010 }
15011 $$emit$$"# DONE"
15012 %}
15013 ins_encode %{
15014 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15015 $tmp$$XMMRegister, false, true, $ktmp$$KRegister);
15016 %}
15017 ins_pipe(pipe_slow);
15018 %}
15019
15020 // Large non-constant length ClearArray for non-AVX512 targets.
15021 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
15022 Universe dummy, rFlagsReg cr)
15023 %{
15024 predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
15025 match(Set dummy (ClearArray (Binary cnt base) val));
15026 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
15027
15028 format %{ $$template
15029 if (UseFastStosb) {
15030 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15031 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15032 } else if (UseXMMForObjInit) {
15033 $$emit$$"movdq $tmp, $val\n\t"
15034 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
15035 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
15036 $$emit$$"jmpq L_zero_64_bytes\n\t"
15037 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15038 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15039 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
15040 $$emit$$"add 0x40,rax\n\t"
15041 $$emit$$"# L_zero_64_bytes:\n\t"
15042 $$emit$$"sub 0x8,rcx\n\t"
15043 $$emit$$"jge L_loop\n\t"
15044 $$emit$$"add 0x4,rcx\n\t"
15045 $$emit$$"jl L_tail\n\t"
15046 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15047 $$emit$$"add 0x20,rax\n\t"
15048 $$emit$$"sub 0x4,rcx\n\t"
15049 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15050 $$emit$$"add 0x4,rcx\n\t"
15051 $$emit$$"jle L_end\n\t"
15052 $$emit$$"dec rcx\n\t"
15053 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15054 $$emit$$"vmovq xmm0,(rax)\n\t"
15055 $$emit$$"add 0x8,rax\n\t"
15056 $$emit$$"dec rcx\n\t"
15057 $$emit$$"jge L_sloop\n\t"
15058 $$emit$$"# L_end:\n\t"
15059 } else {
15060 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15061 }
15062 %}
15063 ins_encode %{
15064 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15065 $tmp$$XMMRegister, true, false);
15066 %}
15067 ins_pipe(pipe_slow);
15068 %}
15069
15070 instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
15071 Universe dummy, rFlagsReg cr)
15072 %{
15073 predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
15074 match(Set dummy (ClearArray (Binary cnt base) val));
15075 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
15076
15077 format %{ $$template
15078 if (UseXMMForObjInit) {
15079 $$emit$$"movdq $tmp, $val\n\t"
15080 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
15081 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
15082 $$emit$$"jmpq L_zero_64_bytes\n\t"
15083 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15084 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15085 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
15086 $$emit$$"add 0x40,rax\n\t"
15087 $$emit$$"# L_zero_64_bytes:\n\t"
15088 $$emit$$"sub 0x8,rcx\n\t"
15089 $$emit$$"jge L_loop\n\t"
15090 $$emit$$"add 0x4,rcx\n\t"
15091 $$emit$$"jl L_tail\n\t"
15092 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15093 $$emit$$"add 0x20,rax\n\t"
15094 $$emit$$"sub 0x4,rcx\n\t"
15095 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15096 $$emit$$"add 0x4,rcx\n\t"
15097 $$emit$$"jle L_end\n\t"
15098 $$emit$$"dec rcx\n\t"
15099 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15100 $$emit$$"vmovq xmm0,(rax)\n\t"
15101 $$emit$$"add 0x8,rax\n\t"
15102 $$emit$$"dec rcx\n\t"
15103 $$emit$$"jge L_sloop\n\t"
15104 $$emit$$"# L_end:\n\t"
15105 } else {
15106 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15107 }
15108 %}
15109 ins_encode %{
15110 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15111 $tmp$$XMMRegister, true, true);
15112 %}
15113 ins_pipe(pipe_slow);
15114 %}
15115
15116 // Large non-constant length ClearArray for AVX512 targets.
15117 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
15118 Universe dummy, rFlagsReg cr)
15119 %{
15120 predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
15121 match(Set dummy (ClearArray (Binary cnt base) val));
15122 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
15123
15124 format %{ $$template
15125 if (UseFastStosb) {
15126 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15127 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15128 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15129 } else if (UseXMMForObjInit) {
15130 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
15131 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
15132 $$emit$$"jmpq L_zero_64_bytes\n\t"
15133 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15134 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15135 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
15136 $$emit$$"add 0x40,rax\n\t"
15137 $$emit$$"# L_zero_64_bytes:\n\t"
15138 $$emit$$"sub 0x8,rcx\n\t"
15139 $$emit$$"jge L_loop\n\t"
15140 $$emit$$"add 0x4,rcx\n\t"
15141 $$emit$$"jl L_tail\n\t"
15142 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15143 $$emit$$"add 0x20,rax\n\t"
15144 $$emit$$"sub 0x4,rcx\n\t"
15145 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15146 $$emit$$"add 0x4,rcx\n\t"
15147 $$emit$$"jle L_end\n\t"
15148 $$emit$$"dec rcx\n\t"
15149 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15150 $$emit$$"vmovq xmm0,(rax)\n\t"
15151 $$emit$$"add 0x8,rax\n\t"
15152 $$emit$$"dec rcx\n\t"
15153 $$emit$$"jge L_sloop\n\t"
15154 $$emit$$"# L_end:\n\t"
15155 } else {
15156 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15157 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15158 }
15159 %}
15160 ins_encode %{
15161 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15162 $tmp$$XMMRegister, true, false, $ktmp$$KRegister);
15163 %}
15164 ins_pipe(pipe_slow);
15165 %}
15166
15167 instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
15168 Universe dummy, rFlagsReg cr)
15169 %{
15170 predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
15171 match(Set dummy (ClearArray (Binary cnt base) val));
15172 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
15173
15174 format %{ $$template
15175 if (UseFastStosb) {
15176 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15177 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15178 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15179 } else if (UseXMMForObjInit) {
15180 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
15181 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
15182 $$emit$$"jmpq L_zero_64_bytes\n\t"
15183 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15184 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15185 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
15186 $$emit$$"add 0x40,rax\n\t"
15187 $$emit$$"# L_zero_64_bytes:\n\t"
15188 $$emit$$"sub 0x8,rcx\n\t"
15189 $$emit$$"jge L_loop\n\t"
15190 $$emit$$"add 0x4,rcx\n\t"
15191 $$emit$$"jl L_tail\n\t"
15192 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15193 $$emit$$"add 0x20,rax\n\t"
15194 $$emit$$"sub 0x4,rcx\n\t"
15195 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15196 $$emit$$"add 0x4,rcx\n\t"
15197 $$emit$$"jle L_end\n\t"
15198 $$emit$$"dec rcx\n\t"
15199 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15200 $$emit$$"vmovq xmm0,(rax)\n\t"
15201 $$emit$$"add 0x8,rax\n\t"
15202 $$emit$$"dec rcx\n\t"
15203 $$emit$$"jge L_sloop\n\t"
15204 $$emit$$"# L_end:\n\t"
15205 } else {
15206 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15207 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15208 }
15209 %}
15210 ins_encode %{
15211 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15212 $tmp$$XMMRegister, true, true, $ktmp$$KRegister);
15213 %}
15214 ins_pipe(pipe_slow);
15215 %}
15216
15217 // Small constant length ClearArray for AVX512 targets.
15218 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rax_RegL val, kReg ktmp, Universe dummy, rFlagsReg cr)
15219 %{
15220 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
15221 ((MaxVectorSize >= 32) && VM_Version::supports_avx512vl()));
15222 match(Set dummy (ClearArray (Binary cnt base) val));
15223 ins_cost(100);
15224 effect(TEMP tmp, USE_KILL val, TEMP ktmp, KILL cr);
15225 format %{ "clear_mem_imm $base , $cnt \n\t" %}
15226 ins_encode %{
15227 __ clear_mem($base$$Register, $cnt$$constant, $val$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
15228 %}
15229 ins_pipe(pipe_slow);
15230 %}
15231
15232 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15233 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15234 %{
15235 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
15236 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15237 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15238
15239 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15240 ins_encode %{
15241 __ string_compare($str1$$Register, $str2$$Register,
15242 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15243 $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
15244 %}
15245 ins_pipe( pipe_slow );
15246 %}
15247
17085 effect(USE meth);
17086
17087 ins_cost(300);
17088 format %{ "call_leaf,runtime " %}
17089 ins_encode(clear_avx, Java_To_Runtime(meth));
17090 ins_pipe(pipe_slow);
17091 %}
17092
17093 // Call runtime without safepoint and with vector arguments
17094 instruct CallLeafDirectVector(method meth)
17095 %{
17096 match(CallLeafVector);
17097 effect(USE meth);
17098
17099 ins_cost(300);
17100 format %{ "call_leaf,vector " %}
17101 ins_encode(Java_To_Runtime(meth));
17102 ins_pipe(pipe_slow);
17103 %}
17104
17105 // Call runtime without safepoint
17106 // entry point is null, target holds the address to call
17107 instruct CallLeafNoFPInDirect(rRegP target)
17108 %{
17109 predicate(n->as_Call()->entry_point() == nullptr);
17110 match(CallLeafNoFP target);
17111
17112 ins_cost(300);
17113 format %{ "call_leaf_nofp,runtime indirect " %}
17114 ins_encode %{
17115 __ call($target$$Register);
17116 %}
17117
17118 ins_pipe(pipe_slow);
17119 %}
17120
17121 // Call runtime without safepoint
17122 instruct CallLeafNoFPDirect(method meth)
17123 %{
17124 predicate(n->as_Call()->entry_point() != nullptr);
17125 match(CallLeafNoFP);
17126 effect(USE meth);
17127
17128 ins_cost(300);
17129 format %{ "call_leaf_nofp,runtime " %}
17130 ins_encode(clear_avx, Java_To_Runtime(meth));
17131 ins_pipe(pipe_slow);
17132 %}
17133
17134 // Return Instruction
17135 // Remove the return address & jump to it.
17136 // Notice: We always emit a nop after a ret to make sure there is room
17137 // for safepoint patching
17138 instruct Ret()
17139 %{
17140 match(Return);
17141
17142 format %{ "ret" %}
17143 ins_encode %{
17144 __ ret(0);
|