< prev index next >

src/hotspot/cpu/x86/macroAssembler_x86.cpp

Print this page

8213       evmovdqub(dst, kmask, src, true, vector_len);
8214       break;
8215     case T_CHAR:
8216     case T_SHORT:
8217       evmovdquw(dst, kmask, src, true, vector_len);
8218       break;
8219     case T_INT:
8220     case T_FLOAT:
8221       evmovdqul(dst, kmask, src, true, vector_len);
8222       break;
8223     case T_LONG:
8224     case T_DOUBLE:
8225       evmovdquq(dst, kmask, src, true, vector_len);
8226       break;
8227     default:
8228       fatal("Unexpected type argument %s", type2name(type));
8229       break;
8230   }
8231 }
8232 





















































































































































































































































































































































































8233 #if COMPILER2_OR_JVMCI
8234 
8235 
8236 // Set memory operation for length "less than" 64 bytes.
8237 void MacroAssembler::fill64_masked_avx(uint shift, Register dst, int disp,
8238                                        XMMRegister xmm, KRegister mask, Register length,
8239                                        Register temp, bool use64byteVector) {
8240   assert(MaxVectorSize >= 32, "vector length should be >= 32");
8241   assert(shift != 0, "shift value should be 1 (short),2(int) or 3(long)");
8242   BasicType type[] = { T_BYTE, T_SHORT,  T_INT,   T_LONG};
8243   if (!use64byteVector) {
8244     fill32_avx(dst, disp, xmm);
8245     subptr(length, 32 >> shift);
8246     fill32_masked_avx(shift, dst, disp + 32, xmm, mask, length, temp);
8247   } else {
8248     assert(MaxVectorSize == 64, "vector length != 64");
8249     movl(temp, 1);
8250     shlxl(temp, temp, length);
8251     subptr(temp, 1);
8252     kmovwl(mask, temp);

8213       evmovdqub(dst, kmask, src, true, vector_len);
8214       break;
8215     case T_CHAR:
8216     case T_SHORT:
8217       evmovdquw(dst, kmask, src, true, vector_len);
8218       break;
8219     case T_INT:
8220     case T_FLOAT:
8221       evmovdqul(dst, kmask, src, true, vector_len);
8222       break;
8223     case T_LONG:
8224     case T_DOUBLE:
8225       evmovdquq(dst, kmask, src, true, vector_len);
8226       break;
8227     default:
8228       fatal("Unexpected type argument %s", type2name(type));
8229       break;
8230   }
8231 }
8232 
8233 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
8234   switch(masklen) {
8235     case 2:
8236        knotbl(dst, src);
8237        movl(rtmp, 3);
8238        kmovbl(ktmp, rtmp);
8239        kandbl(dst, ktmp, dst);
8240        break;
8241     case 4:
8242        knotbl(dst, src);
8243        movl(rtmp, 15);
8244        kmovbl(ktmp, rtmp);
8245        kandbl(dst, ktmp, dst);
8246        break;
8247     case 8:
8248        knotbl(dst, src);
8249        break;
8250     case 16:
8251        knotwl(dst, src);
8252        break;
8253     case 32:
8254        knotdl(dst, src);
8255        break;
8256     case 64:
8257        knotql(dst, src);
8258        break;
8259     default:
8260       fatal("Unexpected vector length %d", masklen);
8261       break;
8262   }
8263 }
8264 
8265 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8266   switch(type) {
8267     case T_BOOLEAN:
8268     case T_BYTE:
8269        kandbl(dst, src1, src2);
8270        break;
8271     case T_CHAR:
8272     case T_SHORT:
8273        kandwl(dst, src1, src2);
8274        break;
8275     case T_INT:
8276     case T_FLOAT:
8277        kanddl(dst, src1, src2);
8278        break;
8279     case T_LONG:
8280     case T_DOUBLE:
8281        kandql(dst, src1, src2);
8282        break;
8283     default:
8284       fatal("Unexpected type argument %s", type2name(type));
8285       break;
8286   }
8287 }
8288 
8289 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8290   switch(type) {
8291     case T_BOOLEAN:
8292     case T_BYTE:
8293        korbl(dst, src1, src2);
8294        break;
8295     case T_CHAR:
8296     case T_SHORT:
8297        korwl(dst, src1, src2);
8298        break;
8299     case T_INT:
8300     case T_FLOAT:
8301        kordl(dst, src1, src2);
8302        break;
8303     case T_LONG:
8304     case T_DOUBLE:
8305        korql(dst, src1, src2);
8306        break;
8307     default:
8308       fatal("Unexpected type argument %s", type2name(type));
8309       break;
8310   }
8311 }
8312 
8313 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
8314   switch(type) {
8315     case T_BOOLEAN:
8316     case T_BYTE:
8317        kxorbl(dst, src1, src2);
8318        break;
8319     case T_CHAR:
8320     case T_SHORT:
8321        kxorwl(dst, src1, src2);
8322        break;
8323     case T_INT:
8324     case T_FLOAT:
8325        kxordl(dst, src1, src2);
8326        break;
8327     case T_LONG:
8328     case T_DOUBLE:
8329        kxorql(dst, src1, src2);
8330        break;
8331     default:
8332       fatal("Unexpected type argument %s", type2name(type));
8333       break;
8334   }
8335 }
8336 
8337 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8338   switch(type) {
8339     case T_BOOLEAN:
8340     case T_BYTE:
8341       evpermb(dst, mask, nds, src, merge, vector_len); break;
8342     case T_CHAR:
8343     case T_SHORT:
8344       evpermw(dst, mask, nds, src, merge, vector_len); break;
8345     case T_INT:
8346     case T_FLOAT:
8347       evpermd(dst, mask, nds, src, merge, vector_len); break;
8348     case T_LONG:
8349     case T_DOUBLE:
8350       evpermq(dst, mask, nds, src, merge, vector_len); break;
8351     default:
8352       fatal("Unexpected type argument %s", type2name(type)); break;
8353   }
8354 }
8355 
8356 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8357   switch(type) {
8358     case T_BOOLEAN:
8359     case T_BYTE:
8360       evpermb(dst, mask, nds, src, merge, vector_len); break;
8361     case T_CHAR:
8362     case T_SHORT:
8363       evpermw(dst, mask, nds, src, merge, vector_len); break;
8364     case T_INT:
8365     case T_FLOAT:
8366       evpermd(dst, mask, nds, src, merge, vector_len); break;
8367     case T_LONG:
8368     case T_DOUBLE:
8369       evpermq(dst, mask, nds, src, merge, vector_len); break;
8370     default:
8371       fatal("Unexpected type argument %s", type2name(type)); break;
8372   }
8373 }
8374 
8375 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8376   switch(type) {
8377     case T_BYTE:
8378       evpminsb(dst, mask, nds, src, merge, vector_len); break;
8379     case T_SHORT:
8380       evpminsw(dst, mask, nds, src, merge, vector_len); break;
8381     case T_INT:
8382       evpminsd(dst, mask, nds, src, merge, vector_len); break;
8383     case T_LONG:
8384       evpminsq(dst, mask, nds, src, merge, vector_len); break;
8385     default:
8386       fatal("Unexpected type argument %s", type2name(type)); break;
8387   }
8388 }
8389 
8390 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8391   switch(type) {
8392     case T_BYTE:
8393       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
8394     case T_SHORT:
8395       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
8396     case T_INT:
8397       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
8398     case T_LONG:
8399       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
8400     default:
8401       fatal("Unexpected type argument %s", type2name(type)); break;
8402   }
8403 }
8404 
8405 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8406   switch(type) {
8407     case T_BYTE:
8408       evpminsb(dst, mask, nds, src, merge, vector_len); break;
8409     case T_SHORT:
8410       evpminsw(dst, mask, nds, src, merge, vector_len); break;
8411     case T_INT:
8412       evpminsd(dst, mask, nds, src, merge, vector_len); break;
8413     case T_LONG:
8414       evpminsq(dst, mask, nds, src, merge, vector_len); break;
8415     default:
8416       fatal("Unexpected type argument %s", type2name(type)); break;
8417   }
8418 }
8419 
8420 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8421   switch(type) {
8422     case T_BYTE:
8423       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
8424     case T_SHORT:
8425       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
8426     case T_INT:
8427       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
8428     case T_LONG:
8429       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
8430     default:
8431       fatal("Unexpected type argument %s", type2name(type)); break;
8432   }
8433 }
8434 
8435 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8436   switch(type) {
8437     case T_INT:
8438       evpxord(dst, mask, nds, src, merge, vector_len); break;
8439     case T_LONG:
8440       evpxorq(dst, mask, nds, src, merge, vector_len); break;
8441     default:
8442       fatal("Unexpected type argument %s", type2name(type)); break;
8443   }
8444 }
8445 
8446 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8447   switch(type) {
8448     case T_INT:
8449       evpxord(dst, mask, nds, src, merge, vector_len); break;
8450     case T_LONG:
8451       evpxorq(dst, mask, nds, src, merge, vector_len); break;
8452     default:
8453       fatal("Unexpected type argument %s", type2name(type)); break;
8454   }
8455 }
8456 
8457 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8458   switch(type) {
8459     case T_INT:
8460       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
8461     case T_LONG:
8462       evporq(dst, mask, nds, src, merge, vector_len); break;
8463     default:
8464       fatal("Unexpected type argument %s", type2name(type)); break;
8465   }
8466 }
8467 
8468 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8469   switch(type) {
8470     case T_INT:
8471       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
8472     case T_LONG:
8473       evporq(dst, mask, nds, src, merge, vector_len); break;
8474     default:
8475       fatal("Unexpected type argument %s", type2name(type)); break;
8476   }
8477 }
8478 
8479 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8480   switch(type) {
8481     case T_INT:
8482       evpandd(dst, mask, nds, src, merge, vector_len); break;
8483     case T_LONG:
8484       evpandq(dst, mask, nds, src, merge, vector_len); break;
8485     default:
8486       fatal("Unexpected type argument %s", type2name(type)); break;
8487   }
8488 }
8489 
8490 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8491   switch(type) {
8492     case T_INT:
8493       evpandd(dst, mask, nds, src, merge, vector_len); break;
8494     case T_LONG:
8495       evpandq(dst, mask, nds, src, merge, vector_len); break;
8496     default:
8497       fatal("Unexpected type argument %s", type2name(type)); break;
8498   }
8499 }
8500 
8501 void MacroAssembler::anytrue(Register dst, uint masklen, KRegister src1, KRegister src2) {
8502    masklen = masklen < 8 ? 8 : masklen;
8503    ktest(masklen, src1, src2);
8504    setb(Assembler::notZero, dst);
8505    movzbl(dst, dst);
8506 }
8507 
8508 void MacroAssembler::alltrue(Register dst, uint masklen, KRegister src1, KRegister src2, KRegister kscratch) {
8509   if (masklen < 8) {
8510     knotbl(kscratch, src2);
8511     kortestbl(src1, kscratch);
8512     setb(Assembler::carrySet, dst);
8513     movzbl(dst, dst);
8514   } else {
8515     ktest(masklen, src1, src2);
8516     setb(Assembler::carrySet, dst);
8517     movzbl(dst, dst);
8518   }
8519 }
8520 
8521 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
8522   switch(masklen) {
8523     case 8:
8524        kortestbl(src1, src2);
8525        break;
8526     case 16:
8527        kortestwl(src1, src2);
8528        break;
8529     case 32:
8530        kortestdl(src1, src2);
8531        break;
8532     case 64:
8533        kortestql(src1, src2);
8534        break;
8535     default:
8536       fatal("Unexpected mask length %d", masklen);
8537       break;
8538   }
8539 }
8540 
8541 
8542 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
8543   switch(masklen)  {
8544     case 8:
8545        ktestbl(src1, src2);
8546        break;
8547     case 16:
8548        ktestwl(src1, src2);
8549        break;
8550     case 32:
8551        ktestdl(src1, src2);
8552        break;
8553     case 64:
8554        ktestql(src1, src2);
8555        break;
8556     default:
8557       fatal("Unexpected mask length %d", masklen);
8558       break;
8559   }
8560 }
8561 
8562 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
8563   switch(type) {
8564     case T_INT:
8565       evprold(dst, mask, src, shift, merge, vlen_enc); break;
8566     case T_LONG:
8567       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
8568     default:
8569       fatal("Unexpected type argument %s", type2name(type)); break;
8570       break;
8571   }
8572 }
8573 
8574 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
8575   switch(type) {
8576     case T_INT:
8577       evprord(dst, mask, src, shift, merge, vlen_enc); break;
8578     case T_LONG:
8579       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
8580     default:
8581       fatal("Unexpected type argument %s", type2name(type)); break;
8582   }
8583 }
8584 
8585 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
8586   switch(type) {
8587     case T_INT:
8588       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
8589     case T_LONG:
8590       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
8591     default:
8592       fatal("Unexpected type argument %s", type2name(type)); break;
8593   }
8594 }
8595 
8596 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
8597   switch(type) {
8598     case T_INT:
8599       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
8600     case T_LONG:
8601       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
8602     default:
8603       fatal("Unexpected type argument %s", type2name(type)); break;
8604   }
8605 }
8606 #if COMPILER2_OR_JVMCI
8607 
8608 
8609 // Set memory operation for length "less than" 64 bytes.
8610 void MacroAssembler::fill64_masked_avx(uint shift, Register dst, int disp,
8611                                        XMMRegister xmm, KRegister mask, Register length,
8612                                        Register temp, bool use64byteVector) {
8613   assert(MaxVectorSize >= 32, "vector length should be >= 32");
8614   assert(shift != 0, "shift value should be 1 (short),2(int) or 3(long)");
8615   BasicType type[] = { T_BYTE, T_SHORT,  T_INT,   T_LONG};
8616   if (!use64byteVector) {
8617     fill32_avx(dst, disp, xmm);
8618     subptr(length, 32 >> shift);
8619     fill32_masked_avx(shift, dst, disp + 32, xmm, mask, length, temp);
8620   } else {
8621     assert(MaxVectorSize == 64, "vector length != 64");
8622     movl(temp, 1);
8623     shlxl(temp, temp, length);
8624     subptr(temp, 1);
8625     kmovwl(mask, temp);
< prev index next >