< prev index next >

src/hotspot/cpu/aarch64/aarch64_neon.ad

Print this page

5666   format %{ "frint  $dst, $src, $rmode" %}
5667   ins_encode %{
5668     switch ($rmode$$constant) {
5669       case RoundDoubleModeNode::rmode_rint:
5670         __ frintn(as_FloatRegister($dst$$reg), __ T2D,
5671                   as_FloatRegister($src$$reg));
5672         break;
5673       case RoundDoubleModeNode::rmode_floor:
5674         __ frintm(as_FloatRegister($dst$$reg), __ T2D,
5675                   as_FloatRegister($src$$reg));
5676         break;
5677       case RoundDoubleModeNode::rmode_ceil:
5678         __ frintp(as_FloatRegister($dst$$reg), __ T2D,
5679                   as_FloatRegister($src$$reg));
5680         break;
5681     }
5682   %}
5683   ins_pipe(vdop_fp128);
5684 %}
5685 
5686 instruct vpopcount4I(vecX dst, vecX src) %{
5687   predicate(UsePopCountInstruction && n->as_Vector()->length() == 4);
5688   match(Set dst (PopCountVI src));
5689   format %{
5690     "cnt     $dst, $src\t# vector (16B)\n\t"
5691     "uaddlp  $dst, $dst\t# vector (16B)\n\t"
5692     "uaddlp  $dst, $dst\t# vector (8H)"











5693   %}








5694   ins_encode %{




























5695     __ cnt(as_FloatRegister($dst$$reg), __ T16B,
5696            as_FloatRegister($src$$reg));
5697     __ uaddlp(as_FloatRegister($dst$$reg), __ T16B,
5698               as_FloatRegister($dst$$reg));
5699     __ uaddlp(as_FloatRegister($dst$$reg), __ T8H,
5700               as_FloatRegister($dst$$reg));




5701   %}
5702   ins_pipe(pipe_class_default);
5703 %}
5704 
5705 instruct vpopcount2I(vecD dst, vecD src) %{
5706   predicate(UsePopCountInstruction && n->as_Vector()->length() == 2);
5707   match(Set dst (PopCountVI src));
5708   format %{
5709     "cnt     $dst, $src\t# vector (8B)\n\t"
5710     "uaddlp  $dst, $dst\t# vector (8B)\n\t"
5711     "uaddlp  $dst, $dst\t# vector (4H)"
5712   %}
5713   ins_encode %{
5714     __ cnt(as_FloatRegister($dst$$reg), __ T8B,

5715            as_FloatRegister($src$$reg));
5716     __ uaddlp(as_FloatRegister($dst$$reg), __ T8B,
5717               as_FloatRegister($dst$$reg));
5718     __ uaddlp(as_FloatRegister($dst$$reg), __ T4H,


5719               as_FloatRegister($dst$$reg));
5720   %}
5721   ins_pipe(pipe_class_default);
5722 %}
5723 
5724 // vector mask reductions
5725 
5726 instruct vmask_truecount8B(iRegINoSp dst, vecD src, vecD tmp) %{
5727   predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
5728   match(Set dst (VectorMaskTrueCount src));
5729   effect(TEMP tmp);
5730   ins_cost(2 * INSN_COST);
5731   format %{ "addv $tmp, $src\n\t"
5732             "umov $dst, $tmp, B, 0\t# vector (8B)" %}
5733   ins_encode %{
5734     // Input "src" is a vector of boolean represented as bytes with
5735     // 0x00/0x01 as element values.
5736     __ addv(as_FloatRegister($tmp$$reg), __ T8B, as_FloatRegister($src$$reg));
5737     __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
5738   %}

5904   ins_pipe(pipe_slow);
5905 %}
5906 
5907 instruct vmask_tolong16B(iRegLNoSp dst, vecX src) %{
5908   match(Set dst (VectorMaskToLong src));
5909   ins_cost(11 * INSN_COST);
5910   format %{ "vmask_tolong $dst, $src\t# convert mask to long (16B)" %}
5911   ins_encode %{
5912     // Input "src" is a vector of boolean represented as
5913     // bytes with 0x00/0x01 as element values.
5914 
5915     __ umov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ D, 0);
5916     __ umov(rscratch1, as_FloatRegister($src$$reg), __ D, 1);
5917     __ bytemask_compress(as_Register($dst$$reg));
5918     __ bytemask_compress(rscratch1);
5919     __ orr(as_Register($dst$$reg), as_Register($dst$$reg),
5920            rscratch1, Assembler::LSL, 8);
5921   %}
5922   ins_pipe(pipe_slow);
5923 %}

































































































































5666   format %{ "frint  $dst, $src, $rmode" %}
5667   ins_encode %{
5668     switch ($rmode$$constant) {
5669       case RoundDoubleModeNode::rmode_rint:
5670         __ frintn(as_FloatRegister($dst$$reg), __ T2D,
5671                   as_FloatRegister($src$$reg));
5672         break;
5673       case RoundDoubleModeNode::rmode_floor:
5674         __ frintm(as_FloatRegister($dst$$reg), __ T2D,
5675                   as_FloatRegister($src$$reg));
5676         break;
5677       case RoundDoubleModeNode::rmode_ceil:
5678         __ frintp(as_FloatRegister($dst$$reg), __ T2D,
5679                   as_FloatRegister($src$$reg));
5680         break;
5681     }
5682   %}
5683   ins_pipe(vdop_fp128);
5684 %}
5685 
5686 instruct vpopcountID(vecD dst, vecD src) %{
5687   predicate(n->as_Vector()->length_in_bytes() < 16);
5688   match(Set dst (PopCountVI src));
5689   ins_cost(3 * INSN_COST);
5690   format %{ "vpopcountI  $dst, $src\t# vector (8B/4H/2S)" %}
5691   ins_encode %{
5692     assert(UsePopCountInstruction, "unsupported");
5693     BasicType bt = Matcher::vector_element_basic_type(this);
5694     __ cnt(as_FloatRegister($dst$$reg), __ T8B,
5695            as_FloatRegister($src$$reg));
5696     if (bt == T_SHORT || bt == T_INT) {
5697       __ uaddlp(as_FloatRegister($dst$$reg), __ T8B,
5698                 as_FloatRegister($dst$$reg));
5699     }
5700     if (bt == T_INT) {
5701       __ uaddlp(as_FloatRegister($dst$$reg), __ T4H,
5702                 as_FloatRegister($dst$$reg));
5703     }
5704   %}
5705   ins_pipe(pipe_class_default);
5706 %}
5707 
5708 instruct vpopcountIX(vecX dst, vecX src) %{
5709   predicate(n->as_Vector()->length_in_bytes() == 16);
5710   match(Set dst (PopCountVI src));
5711   ins_cost(3 * INSN_COST);
5712   format %{ "vpopcountI  $dst, $src\t# vector (16B/8H/4S)" %}
5713   ins_encode %{
5714     assert(UsePopCountInstruction, "unsupported");
5715     BasicType bt = Matcher::vector_element_basic_type(this);
5716     __ cnt(as_FloatRegister($dst$$reg), __ T16B,
5717            as_FloatRegister($src$$reg));
5718     if (bt == T_SHORT || bt == T_INT) {
5719       __ uaddlp(as_FloatRegister($dst$$reg), __ T16B,
5720                 as_FloatRegister($dst$$reg));
5721     }
5722     if (bt == T_INT) {
5723       __ uaddlp(as_FloatRegister($dst$$reg), __ T8H,
5724                 as_FloatRegister($dst$$reg));
5725     }
5726   %}
5727   ins_pipe(pipe_class_default);
5728 %}
5729 
5730 // If the PopCountVL is generated by auto-vectorization, the dst basic
5731 // type is T_INT. And once we have unified the type definition for
5732 // Vector API and auto-vectorization, this rule can be merged with
5733 // "vpopcountLX" rule.
5734 instruct vpopcountLD(vecD dst, vecX src) %{
5735   predicate(n->as_Vector()->length_in_bytes() < 16 &&
5736             n->bottom_type()->is_vect()->element_basic_type() == T_INT);
5737   match(Set dst (PopCountVL src));
5738   ins_cost(5 * INSN_COST);
5739   format %{ "vpopcountL  $dst, $src\t# vector (2S)" %}
5740   ins_encode %{
5741     assert(UsePopCountInstruction, "unsupported");
5742     __ cnt(as_FloatRegister($dst$$reg), __ T16B,
5743            as_FloatRegister($src$$reg));
5744     __ uaddlp(as_FloatRegister($dst$$reg), __ T16B,
5745               as_FloatRegister($dst$$reg));
5746     __ uaddlp(as_FloatRegister($dst$$reg), __ T8H,
5747               as_FloatRegister($dst$$reg));
5748     __ uaddlp(as_FloatRegister($dst$$reg), __ T4S,
5749               as_FloatRegister($dst$$reg));
5750     __ xtn(as_FloatRegister($dst$$reg), __ T2S,
5751            as_FloatRegister($dst$$reg), __ T2D);
5752   %}
5753   ins_pipe(pipe_class_default);
5754 %}
5755 
5756 instruct vpopcountLX(vecX dst, vecX src) %{
5757   predicate(n->as_Vector()->length_in_bytes() == 16 &&
5758             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
5759   match(Set dst (PopCountVL src));
5760   ins_cost(4 * INSN_COST);
5761   format %{ "vpopcountL  $dst, $src\t# vector (2D)" %}


5762   ins_encode %{
5763     assert(UsePopCountInstruction, "unsupported");
5764     __ cnt(as_FloatRegister($dst$$reg), __ T16B,
5765            as_FloatRegister($src$$reg));
5766     __ uaddlp(as_FloatRegister($dst$$reg), __ T16B,
5767               as_FloatRegister($dst$$reg));
5768     __ uaddlp(as_FloatRegister($dst$$reg), __ T8H,
5769               as_FloatRegister($dst$$reg));
5770     __ uaddlp(as_FloatRegister($dst$$reg), __ T4S,
5771               as_FloatRegister($dst$$reg));
5772   %}
5773   ins_pipe(pipe_class_default);
5774 %}
5775 
5776 // vector mask reductions
5777 
5778 instruct vmask_truecount8B(iRegINoSp dst, vecD src, vecD tmp) %{
5779   predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
5780   match(Set dst (VectorMaskTrueCount src));
5781   effect(TEMP tmp);
5782   ins_cost(2 * INSN_COST);
5783   format %{ "addv $tmp, $src\n\t"
5784             "umov $dst, $tmp, B, 0\t# vector (8B)" %}
5785   ins_encode %{
5786     // Input "src" is a vector of boolean represented as bytes with
5787     // 0x00/0x01 as element values.
5788     __ addv(as_FloatRegister($tmp$$reg), __ T8B, as_FloatRegister($src$$reg));
5789     __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
5790   %}

5956   ins_pipe(pipe_slow);
5957 %}
5958 
5959 instruct vmask_tolong16B(iRegLNoSp dst, vecX src) %{
5960   match(Set dst (VectorMaskToLong src));
5961   ins_cost(11 * INSN_COST);
5962   format %{ "vmask_tolong $dst, $src\t# convert mask to long (16B)" %}
5963   ins_encode %{
5964     // Input "src" is a vector of boolean represented as
5965     // bytes with 0x00/0x01 as element values.
5966 
5967     __ umov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ D, 0);
5968     __ umov(rscratch1, as_FloatRegister($src$$reg), __ D, 1);
5969     __ bytemask_compress(as_Register($dst$$reg));
5970     __ bytemask_compress(rscratch1);
5971     __ orr(as_Register($dst$$reg), as_Register($dst$$reg),
5972            rscratch1, Assembler::LSL, 8);
5973   %}
5974   ins_pipe(pipe_slow);
5975 %}
5976 
5977 //------------------------- CountLeadingZerosV -----------------------------
5978 
5979 instruct countLeadingZerosVD(vecD dst, vecD src) %{
5980   predicate(n->as_Vector()->length_in_bytes() == 8);
5981   match(Set dst (CountLeadingZerosV src));
5982   ins_cost(INSN_COST);
5983   format %{ "countLeadingZerosV $dst, $src\t# vector (8B/4H/2S)" %}
5984   ins_encode %{
5985     BasicType bt = Matcher::vector_element_basic_type(this);
5986     Assembler::SIMD_Arrangement size = __ esize2arrangement((unsigned)type2aelembytes(bt), false);
5987     __ clz(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg));
5988   %}
5989   ins_pipe(pipe_slow);
5990 %}
5991 
5992 instruct countLeadingZerosVX(vecX dst, vecX src) %{
5993   predicate(n->as_Vector()->length_in_bytes() == 16);
5994   match(Set dst (CountLeadingZerosV src));
5995   ins_cost(INSN_COST);
5996   format %{ "countLeadingZerosV $dst, $src\t# vector (16B/8H/4S/2D)" %}
5997   ins_encode %{
5998     BasicType bt = Matcher::vector_element_basic_type(this);
5999     Assembler::SIMD_Arrangement size = __ esize2arrangement((unsigned)type2aelembytes(bt), true);
6000     if (bt != T_LONG) {
6001       __ clz(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg));
6002     } else {
6003       __ umov(rscratch1, as_FloatRegister($src$$reg), __ D, 0);
6004       __ clz(rscratch1, rscratch1);
6005       __ mov(as_FloatRegister($dst$$reg), __ D, 0, rscratch1);
6006       __ umov(rscratch1, as_FloatRegister($src$$reg), __ D, 1);
6007       __ clz(rscratch1, rscratch1);
6008       __ mov(as_FloatRegister($dst$$reg), __ D, 1, rscratch1);
6009     }
6010   %}
6011   ins_pipe(pipe_slow);
6012 %}
6013 
6014 //------------------------- CountTrailingZerosV ----------------------------
6015 
6016 instruct countTrailingZerosVD(vecD dst, vecD src) %{
6017   predicate(n->as_Vector()->length_in_bytes() == 8);
6018   match(Set dst (CountTrailingZerosV src));
6019   ins_cost(3 * INSN_COST);
6020   format %{ "countTrailingZerosV $dst, $src\t# vector (8B/4H/2S)" %}
6021   ins_encode %{
6022     BasicType bt = Matcher::vector_element_basic_type(this);
6023     Assembler::SIMD_Arrangement size = __ esize2arrangement((unsigned)type2aelembytes(bt), false);
6024     __ neon_reverse_bits(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, false);
6025     __ clz(as_FloatRegister($dst$$reg), size, as_FloatRegister($dst$$reg));
6026   %}
6027   ins_pipe(pipe_slow);
6028 %}
6029 
6030 instruct countTrailingZerosVX(vecX dst, vecX src) %{
6031   predicate(n->as_Vector()->length_in_bytes() == 16);
6032   match(Set dst (CountTrailingZerosV src));
6033   ins_cost(3 * INSN_COST);
6034   format %{ "countTrailingZerosV $dst, $src\t# vector (16B/8H/4S/2D)" %}
6035   ins_encode %{
6036     BasicType bt = Matcher::vector_element_basic_type(this);
6037     Assembler::SIMD_Arrangement size = __ esize2arrangement((unsigned)type2aelembytes(bt), true);
6038     __ neon_reverse_bits(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, true);
6039     if (bt != T_LONG) {
6040       __ clz(as_FloatRegister($dst$$reg), size, as_FloatRegister($dst$$reg));
6041     } else {
6042       __ umov(rscratch1, as_FloatRegister($dst$$reg), __ D, 0);
6043       __ clz(rscratch1, rscratch1);
6044       __ mov(as_FloatRegister($dst$$reg), __ D, 0, rscratch1);
6045       __ umov(rscratch1, as_FloatRegister($dst$$reg), __ D, 1);
6046       __ clz(rscratch1, rscratch1);
6047       __ mov(as_FloatRegister($dst$$reg), __ D, 1, rscratch1);
6048     }
6049   %}
6050   ins_pipe(pipe_slow);
6051 %}
6052 
6053 //------------------------------ ReverseV -----------------------------------
6054 
6055 instruct vreverseD(vecD dst, vecD src) %{
6056   predicate(n->as_Vector()->length_in_bytes() == 8);
6057   match(Set dst (ReverseV src));
6058   ins_cost(2 * INSN_COST);
6059   format %{ "ReverseV $dst, $src\t# vector (D)" %}
6060   ins_encode %{
6061     BasicType bt = Matcher::vector_element_basic_type(this);
6062     __ neon_reverse_bits(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, false);
6063   %}
6064   ins_pipe(pipe_slow);
6065 %}
6066 
6067 instruct vreverseX(vecX dst, vecX src) %{
6068   predicate(n->as_Vector()->length_in_bytes() == 16);
6069   match(Set dst (ReverseV src));
6070   ins_cost(2 * INSN_COST);
6071   format %{ "ReverseV $dst, $src\t# vector (X)" %}
6072   ins_encode %{
6073     BasicType bt = Matcher::vector_element_basic_type(this);
6074     __ neon_reverse_bits(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, true);
6075   %}
6076   ins_pipe(pipe_slow);
6077 %}
6078 
6079 //---------------------------- ReverseBytesV --------------------------------
6080 
6081 instruct vreverseBytesD(vecD dst, vecD src) %{
6082   predicate(n->as_Vector()->length_in_bytes() == 8);
6083   match(Set dst (ReverseBytesV src));
6084   ins_cost(INSN_COST);
6085   format %{ "ReverseBytesV $dst, $src\t# vector (D)" %}
6086   ins_encode %{
6087     BasicType bt = Matcher::vector_element_basic_type(this);
6088     __ neon_reverse_bytes(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, false);
6089   %}
6090   ins_pipe(pipe_slow);
6091 %}
6092 
6093 instruct vreverseBytesX(vecX dst, vecX src) %{
6094   predicate(n->as_Vector()->length_in_bytes() == 16);
6095   match(Set dst (ReverseBytesV src));
6096   ins_cost(INSN_COST);
6097   format %{ "ReverseBytesV $dst, $src\t# vector (X)" %}
6098   ins_encode %{
6099     BasicType bt = Matcher::vector_element_basic_type(this);
6100     __ neon_reverse_bytes(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, true);
6101   %}
6102   ins_pipe(pipe_slow);
6103 %}
< prev index next >