< prev index next >

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

Print this page
@@ -1459,10 +1459,23 @@
        default: assert(false, "%s", type2name(elem_bt));
      }
    }
  }
  
+ void C2_MacroAssembler::load_vector_mask(KRegister dst, XMMRegister src, XMMRegister xtmp,
+                                          Register tmp, bool novlbwdq, int vlen_enc) {
+   if (novlbwdq) {
+     vpmovsxbd(xtmp, src, vlen_enc);
+     evpcmpd(dst, k0, xtmp, ExternalAddress(StubRoutines::x86::vector_int_mask_cmp_bits()),
+             Assembler::eq, true, vlen_enc, tmp);
+   } else {
+     vpxor(xtmp, xtmp, xtmp, vlen_enc);
+     vpsubb(xtmp, xtmp, src, vlen_enc);
+     evpmovb2m(dst, xtmp, vlen_enc);
+   }
+ }
+ 
  void C2_MacroAssembler::load_iota_indices(XMMRegister dst, Register scratch, int vlen_in_bytes) {
    ExternalAddress addr(StubRoutines::x86::vector_iota_indices());
    if (vlen_in_bytes == 4) {
      movdl(dst, addr);
    } else if (vlen_in_bytes == 8) {

@@ -3825,18 +3838,235 @@
      vpxor(vec1, vec1);
      vpxor(vec2, vec2);
    }
  }
  
+ void C2_MacroAssembler::evmasked_op(int ideal_opc, BasicType eType, KRegister mask, XMMRegister dst,
+                                     XMMRegister src1, int imm8, bool merge, int vlen_enc) {
+   switch(ideal_opc) {
+     case Op_LShiftVS:
+       Assembler::evpsllw(dst, mask, src1, imm8, merge, vlen_enc); break;
+     case Op_LShiftVI:
+       Assembler::evpslld(dst, mask, src1, imm8, merge, vlen_enc); break;
+     case Op_LShiftVL:
+       Assembler::evpsllq(dst, mask, src1, imm8, merge, vlen_enc); break;
+     case Op_RShiftVS:
+       Assembler::evpsraw(dst, mask, src1, imm8, merge, vlen_enc); break;
+     case Op_RShiftVI:
+       Assembler::evpsrad(dst, mask, src1, imm8, merge, vlen_enc); break;
+     case Op_RShiftVL:
+       Assembler::evpsraq(dst, mask, src1, imm8, merge, vlen_enc); break;
+     case Op_URShiftVS:
+       Assembler::evpsrlw(dst, mask, src1, imm8, merge, vlen_enc); break;
+     case Op_URShiftVI:
+       Assembler::evpsrld(dst, mask, src1, imm8, merge, vlen_enc); break;
+     case Op_URShiftVL:
+       Assembler::evpsrlq(dst, mask, src1, imm8, merge, vlen_enc); break;
+     case Op_RotateRightV:
+       evrord(eType, dst, mask, src1, imm8, merge, vlen_enc); break;
+     case Op_RotateLeftV:
+       evrold(eType, dst, mask, src1, imm8, merge, vlen_enc); break;
+     default:
+       fatal("Unsupported masked operation"); break;
+   }
+ }
+ 
+ void C2_MacroAssembler::evmasked_op(int ideal_opc, BasicType eType, KRegister mask, XMMRegister dst,
+                                     XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc,
+                                     bool is_varshift) {
+   switch (ideal_opc) {
+     case Op_AddVB:
+       evpaddb(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AddVS:
+       evpaddw(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AddVI:
+       evpaddd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AddVL:
+       evpaddq(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AddVF:
+       evaddps(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AddVD:
+       evaddpd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVB:
+       evpsubb(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVS:
+       evpsubw(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVI:
+       evpsubd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVL:
+       evpsubq(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVF:
+       evsubps(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVD:
+       evsubpd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MulVS:
+       evpmullw(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MulVI:
+       evpmulld(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MulVL:
+       evpmullq(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MulVF:
+       evmulps(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MulVD:
+       evmulpd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_DivVF:
+       evdivps(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_DivVD:
+       evdivpd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SqrtVF:
+       evsqrtps(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SqrtVD:
+       evsqrtpd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AbsVB:
+       evpabsb(dst, mask, src2, merge, vlen_enc); break;
+     case Op_AbsVS:
+       evpabsw(dst, mask, src2, merge, vlen_enc); break;
+     case Op_AbsVI:
+       evpabsd(dst, mask, src2, merge, vlen_enc); break;
+     case Op_AbsVL:
+       evpabsq(dst, mask, src2, merge, vlen_enc); break;
+     case Op_FmaVF:
+       evpfma213ps(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_FmaVD:
+       evpfma213pd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_VectorRearrange:
+       evperm(eType, dst, mask, src2, src1, merge, vlen_enc); break;
+     case Op_LShiftVS:
+       evpsllw(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
+     case Op_LShiftVI:
+       evpslld(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
+     case Op_LShiftVL:
+       evpsllq(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
+     case Op_RShiftVS:
+       evpsraw(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
+     case Op_RShiftVI:
+       evpsrad(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
+     case Op_RShiftVL:
+       evpsraq(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
+     case Op_URShiftVS:
+       evpsrlw(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
+     case Op_URShiftVI:
+       evpsrld(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
+     case Op_URShiftVL:
+       evpsrlq(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
+     case Op_RotateLeftV:
+       evrold(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_RotateRightV:
+       evrord(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MaxV:
+       evpmaxs(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MinV:
+       evpmins(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_XorV:
+       evxor(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_OrV:
+       evor(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AndV:
+       evand(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     default:
+       fatal("Unsupported masked operation"); break;
+   }
+ }
+ 
+ void C2_MacroAssembler::evmasked_op(int ideal_opc, BasicType eType, KRegister mask, XMMRegister dst,
+                                     XMMRegister src1, Address src2, bool merge, int vlen_enc) {
+   switch (ideal_opc) {
+     case Op_AddVB:
+       evpaddb(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AddVS:
+       evpaddw(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AddVI:
+       evpaddd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AddVL:
+       evpaddq(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AddVF:
+       evaddps(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AddVD:
+       evaddpd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVB:
+       evpsubb(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVS:
+       evpsubw(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVI:
+       evpsubd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVL:
+       evpsubq(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVF:
+       evsubps(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_SubVD:
+       evsubpd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MulVS:
+       evpmullw(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MulVI:
+       evpmulld(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MulVL:
+       evpmullq(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MulVF:
+       evmulps(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MulVD:
+       evmulpd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_DivVF:
+       evdivps(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_DivVD:
+       evdivpd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_FmaVF:
+       evpfma213ps(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_FmaVD:
+       evpfma213pd(dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MaxV:
+       evpmaxs(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_MinV:
+       evpmins(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_XorV:
+       evxor(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_OrV:
+       evor(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     case Op_AndV:
+       evand(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+     default:
+       fatal("Unsupported masked operation"); break;
+   }
+ }
+ 
+ void C2_MacroAssembler::masked_op(int ideal_opc, int mask_len, KRegister dst,
+                                   KRegister src1, KRegister src2) {
+   BasicType etype = T_ILLEGAL;
+   switch(mask_len) {
+     case 2:
+     case 4:
+     case 8:  etype = T_BYTE; break;
+     case 16: etype = T_SHORT; break;
+     case 32: etype = T_INT; break;
+     case 64: etype = T_LONG; break;
+     default: fatal("Unsupported type"); break;
+   }
+   assert(etype != T_ILLEGAL, "");
+   switch(ideal_opc) {
+     case Op_AndVMask:
+       kand(etype, dst, src1, src2); break;
+     case Op_OrVMask:
+       kor(etype, dst, src1, src2); break;
+     case Op_XorVMask:
+       kxor(etype, dst, src1, src2); break;
+     default:
+       fatal("Unsupported masked operation"); break;
+   }
+ }
+ 
  #ifdef _LP64
- void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp,
-                                               Register tmp, KRegister ktmp, int masklen, int vec_enc) {
-   assert(VM_Version::supports_avx512vlbw(), "");
-   vpxor(xtmp, xtmp, xtmp, vec_enc);
-   vpsubb(xtmp, xtmp, mask, vec_enc);
-   evpmovb2m(ktmp, xtmp, vec_enc);
-   kmovql(tmp, ktmp);
+ void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, KRegister mask,
+                                               Register tmp, int masklen, int masksize,
+                                               int vec_enc) {
+   if(VM_Version::supports_avx512bw()) {
+     kmovql(tmp, mask);
+   } else {
+     assert(masklen <= 16, "");
+     kmovwl(tmp, mask);
+   }
+   if (masksize < 16) {
+     andq(tmp, (((jlong)1 << masklen) - 1));
+   }
    switch(opc) {
      case Op_VectorMaskTrueCount:
        popcntq(dst, tmp);
        break;
      case Op_VectorMaskLastTrue:

@@ -3852,16 +4082,17 @@
      default: assert(false, "Unhandled mask operation");
    }
  }
  
  void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp,
-                                               XMMRegister xtmp1, Register tmp, int masklen, int vec_enc) {
+                                               XMMRegister xtmp1, Register tmp, int masklen, int masksize,
+                                               int vec_enc) {
    assert(VM_Version::supports_avx(), "");
    vpxor(xtmp, xtmp, xtmp, vec_enc);
    vpsubb(xtmp, xtmp, mask, vec_enc);
    vpmovmskb(tmp, xtmp, vec_enc);
-   if (masklen < 64) {
+   if (masksize < 16) {
      andq(tmp, (((jlong)1 << masklen) - 1));
    }
    switch(opc) {
      case Op_VectorMaskTrueCount:
        popcntq(dst, tmp);
< prev index next >