1285 int MachNode::compute_padding(int current_offset) const {
1286 if (flags() & Node::PD::Flag_intel_jcc_erratum) {
1287 Compile* C = Compile::current();
1288 PhaseOutput* output = C->output();
1289 Block* block = output->block();
1290 int index = output->index();
1291 return IntelJccErratum::compute_padding(current_offset, this, block, index, C->regalloc());
1292 } else {
1293 return 0;
1294 }
1295 }
1296
1297 // Emit exception handler code.
1298 // Stuff framesize into a register and call a VM stub routine.
1299 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
1300
1301 // Note that the code buffer's insts_mark is always relative to insts.
1302 // That's why we must use the macroassembler to generate a handler.
1303 C2_MacroAssembler _masm(&cbuf);
1304 address base = __ start_a_stub(size_exception_handler());
1305 if (base == NULL) {
1306 ciEnv::current()->record_failure("CodeCache is full");
1307 return 0; // CodeBuffer::expand failed
1308 }
1309 int offset = __ offset();
1310 __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1311 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1312 __ end_a_stub();
1313 return offset;
1314 }
1315
1316 // Emit deopt handler code.
1317 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
1318
1319 // Note that the code buffer's insts_mark is always relative to insts.
1320 // That's why we must use the macroassembler to generate a handler.
1321 C2_MacroAssembler _masm(&cbuf);
1322 address base = __ start_a_stub(size_deopt_handler());
1323 if (base == NULL) {
1324 ciEnv::current()->record_failure("CodeCache is full");
1325 return 0; // CodeBuffer::expand failed
1326 }
1327 int offset = __ offset();
1328
1329 #ifdef _LP64
1330 address the_pc = (address) __ pc();
1331 Label next;
1332 // push a "the_pc" on the stack without destroying any registers
1333 // as they all may be live.
1334
1335 // push address of "next"
1336 __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
1337 __ bind(next);
1338 // adjust it so it matches "the_pc"
1339 __ subptr(Address(rsp, 0), __ offset() - offset);
1340 #else
1341 InternalAddress here(__ pc());
1342 __ pushptr(here.addr(), noreg);
1343 #endif
2160 return new legVecZOper();
2161 }
2162 if (legacy) {
2163 switch (ideal_reg) {
2164 case Op_VecS: return new legVecSOper();
2165 case Op_VecD: return new legVecDOper();
2166 case Op_VecX: return new legVecXOper();
2167 case Op_VecY: return new legVecYOper();
2168 case Op_VecZ: return new legVecZOper();
2169 }
2170 } else {
2171 switch (ideal_reg) {
2172 case Op_VecS: return new vecSOper();
2173 case Op_VecD: return new vecDOper();
2174 case Op_VecX: return new vecXOper();
2175 case Op_VecY: return new vecYOper();
2176 case Op_VecZ: return new vecZOper();
2177 }
2178 }
2179 ShouldNotReachHere();
2180 return NULL;
2181 }
2182
2183 bool Matcher::is_reg2reg_move(MachNode* m) {
2184 switch (m->rule()) {
2185 case MoveVec2Leg_rule:
2186 case MoveLeg2Vec_rule:
2187 case MoveF2VL_rule:
2188 case MoveF2LEG_rule:
2189 case MoveVL2F_rule:
2190 case MoveLEG2F_rule:
2191 case MoveD2VL_rule:
2192 case MoveD2LEG_rule:
2193 case MoveVL2D_rule:
2194 case MoveLEG2D_rule:
2195 return true;
2196 default:
2197 return false;
2198 }
2199 }
2200
2329 }
2330 return false;
2331 }
2332
2333 // This function identifies sub-graphs in which a 'load' node is
2334 // input to two different nodes, and such that it can be matched
2335 // with BMI instructions like blsi, blsr, etc.
2336 // Example : for b = -a[i] & a[i] can be matched to blsi r32, m32.
2337 // The graph is (AndL (SubL Con0 LoadL*) LoadL*), where LoadL*
2338 // refers to the same node.
2339 //
2340 // Match the generic fused operations pattern (op1 (op2 Con{ConType} mop) mop)
2341 // This is a temporary solution until we make DAGs expressible in ADL.
2342 template<typename ConType>
2343 class FusedPatternMatcher {
2344 Node* _op1_node;
2345 Node* _mop_node;
2346 int _con_op;
2347
2348 static int match_next(Node* n, int next_op, int next_op_idx) {
2349 if (n->in(1) == NULL || n->in(2) == NULL) {
2350 return -1;
2351 }
2352
2353 if (next_op_idx == -1) { // n is commutative, try rotations
2354 if (n->in(1)->Opcode() == next_op) {
2355 return 1;
2356 } else if (n->in(2)->Opcode() == next_op) {
2357 return 2;
2358 }
2359 } else {
2360 assert(next_op_idx > 0 && next_op_idx <= 2, "Bad argument index");
2361 if (n->in(next_op_idx)->Opcode() == next_op) {
2362 return next_op_idx;
2363 }
2364 }
2365 return -1;
2366 }
2367
2368 public:
2369 FusedPatternMatcher(Node* op1_node, Node* mop_node, int con_op) :
2396 if (op2_con_idx == -1) {
2397 return false;
2398 }
2399 // Memory operation must be the other edge
2400 int op2_mop_idx = (op2_con_idx & 1) + 1;
2401 // Check that the memory operation is the same node
2402 if (op2_node->in(op2_mop_idx) == _mop_node) {
2403 // Now check the constant
2404 const Type* con_type = op2_node->in(op2_con_idx)->bottom_type();
2405 if (con_type != Type::TOP && ConType::as_self(con_type)->get_con() == con_value) {
2406 return true;
2407 }
2408 }
2409 }
2410 return false;
2411 }
2412 };
2413
2414 static bool is_bmi_pattern(Node* n, Node* m) {
2415 assert(UseBMI1Instructions, "sanity");
2416 if (n != NULL && m != NULL) {
2417 if (m->Opcode() == Op_LoadI) {
2418 FusedPatternMatcher<TypeInt> bmii(n, m, Op_ConI);
2419 return bmii.match(Op_AndI, -1, Op_SubI, 1, 0) ||
2420 bmii.match(Op_AndI, -1, Op_AddI, -1, -1) ||
2421 bmii.match(Op_XorI, -1, Op_AddI, -1, -1);
2422 } else if (m->Opcode() == Op_LoadL) {
2423 FusedPatternMatcher<TypeLong> bmil(n, m, Op_ConL);
2424 return bmil.match(Op_AndL, -1, Op_SubL, 1, 0) ||
2425 bmil.match(Op_AndL, -1, Op_AddL, -1, -1) ||
2426 bmil.match(Op_XorL, -1, Op_AddL, -1, -1);
2427 }
2428 }
2429 return false;
2430 }
2431
2432 // Should the matcher clone input 'm' of node 'n'?
2433 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2434 // If 'n' and 'm' are part of a graph for BMI instruction, clone the input 'm'.
2435 if (UseBMI1Instructions && is_bmi_pattern(n, m)) {
2436 mstack.push(m, Visit);
2762 uint MachBreakpointNode::size(PhaseRegAlloc* ra_) const {
2763 return MachNode::size(ra_);
2764 }
2765
2766 %}
2767
2768 encode %{
2769
2770 enc_class call_epilog %{
2771 C2_MacroAssembler _masm(&cbuf);
2772 if (VerifyStackAtCalls) {
2773 // Check that stack depth is unchanged: find majik cookie on stack
2774 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
2775 Label L;
2776 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
2777 __ jccb(Assembler::equal, L);
2778 // Die if stack mismatch
2779 __ int3();
2780 __ bind(L);
2781 }
2782 %}
2783
2784 %}
2785
2786 // Operands for bound floating pointer register arguments
2787 operand rxmm0() %{
2788 constraint(ALLOC_IN_RC(xmm0_reg));
2789 match(VecX);
2790 format%{%}
2791 interface(REG_INTER);
2792 %}
2793
2794 //----------OPERANDS-----------------------------------------------------------
2795 // Operand definitions must precede instruction definitions for correct parsing
2796 // in the ADLC because operands constitute user defined types which are used in
2797 // instruction definitions.
2798
2799 // Vectors
2800
2801 // Dummy generic vector class. Should be used for all vector operands.
7439 instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
7440 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
7441 match(Set dst (RoundVD src));
7442 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
7443 format %{ "vector_round_long $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
7444 ins_encode %{
7445 int vlen_enc = vector_length_encoding(this);
7446 InternalAddress new_mxcsr = $constantaddress((jint)0x3F80);
7447 __ vector_round_double_evex($dst$$XMMRegister, $src$$XMMRegister,
7448 ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), new_mxcsr, vlen_enc,
7449 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
7450 %}
7451 ins_pipe( pipe_slow );
7452 %}
7453
7454 #endif // _LP64
7455
7456 // --------------------------------- VectorMaskCmp --------------------------------------
7457
7458 instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
7459 predicate(n->bottom_type()->isa_vectmask() == NULL &&
7460 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 8 && // src1
7461 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
7462 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
7463 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7464 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
7465 ins_encode %{
7466 int vlen_enc = vector_length_encoding(this, $src1);
7467 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
7468 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
7469 __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7470 } else {
7471 __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7472 }
7473 %}
7474 ins_pipe( pipe_slow );
7475 %}
7476
7477 instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
7478 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
7479 n->bottom_type()->isa_vectmask() == NULL &&
7480 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
7481 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7482 effect(TEMP ktmp);
7483 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
7484 ins_encode %{
7485 int vlen_enc = Assembler::AVX_512bit;
7486 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
7487 KRegister mask = k0; // The comparison itself is not being masked.
7488 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
7489 __ evcmpps($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7490 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
7491 } else {
7492 __ evcmppd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7493 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
7494 }
7495 %}
7496 ins_pipe( pipe_slow );
7497 %}
7498
7499 instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
7500 predicate(n->bottom_type()->isa_vectmask() &&
7501 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
7502 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7503 format %{ "vector_compare_evex $dst,$src1,$src2,$cond\t!" %}
7504 ins_encode %{
7505 assert(bottom_type()->isa_vectmask(), "TypeVectMask expected");
7506 int vlen_enc = vector_length_encoding(this, $src1);
7507 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
7508 KRegister mask = k0; // The comparison itself is not being masked.
7509 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
7510 __ evcmpps($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7511 } else {
7512 __ evcmppd($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7513 }
7514 %}
7515 ins_pipe( pipe_slow );
7516 %}
7517
7518 instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
7519 predicate(n->bottom_type()->isa_vectmask() == NULL &&
7520 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
7521 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
7522 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
7523 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
7524 (n->in(2)->get_int() == BoolTest::eq ||
7525 n->in(2)->get_int() == BoolTest::lt ||
7526 n->in(2)->get_int() == BoolTest::gt)); // cond
7527 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7528 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
7529 ins_encode %{
7530 int vlen_enc = vector_length_encoding(this, $src1);
7531 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
7532 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
7533 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, cmp, ww, vlen_enc);
7534 %}
7535 ins_pipe( pipe_slow );
7536 %}
7537
7538 instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
7539 predicate(n->bottom_type()->isa_vectmask() == NULL &&
7540 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
7541 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
7542 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
7543 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
7544 (n->in(2)->get_int() == BoolTest::ne ||
7545 n->in(2)->get_int() == BoolTest::le ||
7546 n->in(2)->get_int() == BoolTest::ge)); // cond
7547 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7548 effect(TEMP dst, TEMP xtmp);
7549 format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
7550 ins_encode %{
7551 int vlen_enc = vector_length_encoding(this, $src1);
7552 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
7553 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
7554 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
7555 %}
7556 ins_pipe( pipe_slow );
7557 %}
7558
7559 instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
7560 predicate(n->bottom_type()->isa_vectmask() == NULL &&
7561 Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
7562 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
7563 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
7564 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
7565 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7566 effect(TEMP dst, TEMP xtmp);
7567 format %{ "vector_compareu $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
7568 ins_encode %{
7569 InternalAddress flip_bit = $constantaddress(high_bit_set(Matcher::vector_element_basic_type(this, $src1)));
7570 int vlen_enc = vector_length_encoding(this, $src1);
7571 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
7572 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
7573
7574 if (vlen_enc == Assembler::AVX_128bit) {
7575 __ vmovddup($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
7576 } else {
7577 __ vbroadcastsd($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
7578 }
7579 __ vpxor($dst$$XMMRegister, $xtmp$$XMMRegister, $src1$$XMMRegister, vlen_enc);
7580 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $src2$$XMMRegister, vlen_enc);
7581 __ vpcmpCCW($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
7582 %}
7583 ins_pipe( pipe_slow );
7584 %}
7585
7586 instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
7587 predicate((n->bottom_type()->isa_vectmask() == NULL &&
7588 Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
7589 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
7590 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7591 effect(TEMP ktmp);
7592 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
7593 ins_encode %{
7594 assert(UseAVX > 2, "required");
7595
7596 int vlen_enc = vector_length_encoding(this, $src1);
7597 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
7598 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
7599 KRegister mask = k0; // The comparison itself is not being masked.
7600 bool merge = false;
7601 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
7602
7603 switch (src1_elem_bt) {
7604 case T_INT: {
7605 __ evpcmpd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
7606 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
7607 break;
7783 // --------------------------------- Vector Blend --------------------------------------
7784
7785 instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
7786 predicate(UseAVX == 0);
7787 match(Set dst (VectorBlend (Binary dst src) mask));
7788 format %{ "vector_blend $dst,$src,$mask\t! using $tmp as TEMP" %}
7789 effect(TEMP tmp);
7790 ins_encode %{
7791 assert(UseSSE >= 4, "required");
7792
7793 if ($mask$$XMMRegister != $tmp$$XMMRegister) {
7794 __ movdqu($tmp$$XMMRegister, $mask$$XMMRegister);
7795 }
7796 __ pblendvb($dst$$XMMRegister, $src$$XMMRegister); // uses xmm0 as mask
7797 %}
7798 ins_pipe( pipe_slow );
7799 %}
7800
7801 instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
7802 predicate(UseAVX > 0 &&
7803 n->in(2)->bottom_type()->isa_vectmask() == NULL &&
7804 Matcher::vector_length_in_bytes(n) <= 32 &&
7805 is_integral_type(Matcher::vector_element_basic_type(n)));
7806 match(Set dst (VectorBlend (Binary src1 src2) mask));
7807 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
7808 ins_encode %{
7809 int vlen_enc = vector_length_encoding(this);
7810 __ vpblendvb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
7811 %}
7812 ins_pipe( pipe_slow );
7813 %}
7814
7815 instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
7816 predicate(UseAVX > 0 &&
7817 n->in(2)->bottom_type()->isa_vectmask() == NULL &&
7818 Matcher::vector_length_in_bytes(n) <= 32 &&
7819 !is_integral_type(Matcher::vector_element_basic_type(n)));
7820 match(Set dst (VectorBlend (Binary src1 src2) mask));
7821 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
7822 ins_encode %{
7823 int vlen_enc = vector_length_encoding(this);
7824 __ vblendvps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
7825 %}
7826 ins_pipe( pipe_slow );
7827 %}
7828
7829 instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
7830 predicate(Matcher::vector_length_in_bytes(n) == 64 &&
7831 n->in(2)->bottom_type()->isa_vectmask() == NULL);
7832 match(Set dst (VectorBlend (Binary src1 src2) mask));
7833 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
7834 effect(TEMP ktmp);
7835 ins_encode %{
7836 int vlen_enc = Assembler::AVX_512bit;
7837 BasicType elem_bt = Matcher::vector_element_basic_type(this);
7838 __ evpcmp(elem_bt, $ktmp$$KRegister, k0, $mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), Assembler::eq, vlen_enc, noreg);
7839 __ evpblend(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
7840 %}
7841 ins_pipe( pipe_slow );
7842 %}
7843
7844
7845 instruct evblendvp64_masked(vec dst, vec src1, vec src2, kReg mask) %{
7846 predicate(n->in(2)->bottom_type()->isa_vectmask() &&
7847 (!is_subword_type(Matcher::vector_element_basic_type(n)) ||
7848 VM_Version::supports_avx512bw()));
7849 match(Set dst (VectorBlend (Binary src1 src2) mask));
7850 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
7851 ins_encode %{
8028 %}
8029 ins_pipe( pipe_slow );
8030 %}
8031
8032 instruct ktest_ge8(rFlagsRegU cr, kReg src1, kReg src2) %{
8033 predicate(Matcher::vector_length(n->in(1)) >= 16 ||
8034 (Matcher::vector_length(n->in(1)) == 8 && VM_Version::supports_avx512dq()));
8035 match(Set cr (VectorTest src1 src2));
8036 format %{ "ktest_ge8 $src1, $src2\n\t" %}
8037 ins_encode %{
8038 uint masklen = Matcher::vector_length(this, $src1);
8039 __ kortest(masklen, $src1$$KRegister, $src1$$KRegister);
8040 %}
8041 ins_pipe( pipe_slow );
8042 %}
8043 #endif
8044
8045 //------------------------------------- LoadMask --------------------------------------------
8046
8047 instruct loadMask(legVec dst, legVec src) %{
8048 predicate(n->bottom_type()->isa_vectmask() == NULL && !VM_Version::supports_avx512vlbw());
8049 match(Set dst (VectorLoadMask src));
8050 effect(TEMP dst);
8051 format %{ "vector_loadmask_byte $dst, $src\n\t" %}
8052 ins_encode %{
8053 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
8054 BasicType elem_bt = Matcher::vector_element_basic_type(this);
8055 __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt, true);
8056 %}
8057 ins_pipe( pipe_slow );
8058 %}
8059
8060 instruct loadMask64(kReg dst, vec src, vec xtmp) %{
8061 predicate(n->bottom_type()->isa_vectmask() && !VM_Version::supports_avx512vlbw());
8062 match(Set dst (VectorLoadMask src));
8063 effect(TEMP xtmp);
8064 format %{ "vector_loadmask_64byte $dst, $src\t! using $xtmp as TEMP" %}
8065 ins_encode %{
8066 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
8067 true, Assembler::AVX_512bit);
8068 %}
8069 ins_pipe( pipe_slow );
8070 %}
8071
8072 instruct loadMask_evex(kReg dst, vec src, vec xtmp) %{
8073 predicate(n->bottom_type()->isa_vectmask() && VM_Version::supports_avx512vlbw());
8074 match(Set dst (VectorLoadMask src));
8075 effect(TEMP xtmp);
8076 format %{ "vector_loadmask_byte $dst, $src\t! using $xtmp as TEMP" %}
8077 ins_encode %{
8078 int vlen_enc = vector_length_encoding(in(1));
8079 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
8080 false, vlen_enc);
8081 %}
8082 ins_pipe( pipe_slow );
8083 %}
8084
8085 //------------------------------------- StoreMask --------------------------------------------
8086
8087 instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
8088 predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
8089 match(Set dst (VectorStoreMask src size));
8090 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
8091 ins_encode %{
8092 int vlen = Matcher::vector_length(this);
8093 if (vlen <= 16 && UseAVX <= 2) {
8094 assert(UseSSE >= 3, "required");
8095 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
8096 } else {
8097 assert(UseAVX > 0, "required");
8098 int src_vlen_enc = vector_length_encoding(this, $src);
8099 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
8100 }
8101 %}
8102 ins_pipe( pipe_slow );
8103 %}
8104
8105 instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
8106 predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
8107 match(Set dst (VectorStoreMask src size));
8108 effect(TEMP_DEF dst, TEMP xtmp);
8109 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
8110 ins_encode %{
8111 int vlen_enc = Assembler::AVX_128bit;
8112 int vlen = Matcher::vector_length(this);
8113 if (vlen <= 8) {
8114 assert(UseSSE >= 3, "required");
8115 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
8116 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
8117 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
8118 } else {
8119 assert(UseAVX > 0, "required");
8120 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
8121 __ vpacksswb($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
8122 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
8123 }
8124 %}
8125 ins_pipe( pipe_slow );
8126 %}
8127
8128 instruct vstoreMask4B(vec dst, vec src, vec xtmp, immI_4 size) %{
8129 predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
8130 match(Set dst (VectorStoreMask src size));
8131 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
8132 effect(TEMP_DEF dst, TEMP xtmp);
8133 ins_encode %{
8134 int vlen_enc = Assembler::AVX_128bit;
8135 int vlen = Matcher::vector_length(this);
8136 if (vlen <= 4) {
8137 assert(UseSSE >= 3, "required");
8138 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
8139 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
8140 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
8141 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
8142 } else {
8143 assert(UseAVX > 0, "required");
8144 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
8145 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
8146 __ vpackssdw($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
8147 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
8148 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
8149 }
8169
8170 instruct storeMask8B_avx(vec dst, vec src, immI_8 size, vec vtmp) %{
8171 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 4);
8172 match(Set dst (VectorStoreMask src size));
8173 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s], using $vtmp as TEMP" %}
8174 effect(TEMP_DEF dst, TEMP vtmp);
8175 ins_encode %{
8176 int vlen_enc = Assembler::AVX_128bit;
8177 __ vshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 0x88, Assembler::AVX_256bit);
8178 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
8179 __ vblendps($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0xC, vlen_enc);
8180 __ vpxor($vtmp$$XMMRegister, $vtmp$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
8181 __ vpackssdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
8182 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
8183 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
8184 %}
8185 ins_pipe( pipe_slow );
8186 %}
8187
8188 instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
8189 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
8190 match(Set dst (VectorStoreMask src size));
8191 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
8192 ins_encode %{
8193 int src_vlen_enc = vector_length_encoding(this, $src);
8194 int dst_vlen_enc = vector_length_encoding(this);
8195 if (!VM_Version::supports_avx512vl()) {
8196 src_vlen_enc = Assembler::AVX_512bit;
8197 }
8198 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
8199 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
8200 %}
8201 ins_pipe( pipe_slow );
8202 %}
8203
8204 instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
8205 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
8206 match(Set dst (VectorStoreMask src size));
8207 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
8208 ins_encode %{
8209 int src_vlen_enc = vector_length_encoding(this, $src);
8210 int dst_vlen_enc = vector_length_encoding(this);
8211 if (!VM_Version::supports_avx512vl()) {
8212 src_vlen_enc = Assembler::AVX_512bit;
8213 }
8214 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
8215 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
8216 %}
8217 ins_pipe( pipe_slow );
8218 %}
8219
8220 instruct vstoreMask_evex_vectmask(vec dst, kReg mask, immI size) %{
8221 predicate(n->in(1)->bottom_type()->isa_vectmask() && !VM_Version::supports_avx512vlbw());
8222 match(Set dst (VectorStoreMask mask size));
8223 effect(TEMP_DEF dst);
8224 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
8225 ins_encode %{
9007 %}
9008
9009 instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
9010 predicate(n->in(1)->bottom_type()->isa_vectmask());
9011 match(Set dst (VectorMaskToLong mask));
9012 effect(TEMP dst, KILL cr);
9013 format %{ "vector_tolong_evex $dst, $mask \t! vector mask tolong" %}
9014 ins_encode %{
9015 int opcode = this->ideal_Opcode();
9016 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9017 int mask_len = Matcher::vector_length(this, $mask);
9018 int mask_size = mask_len * type2aelembytes(mbt);
9019 int vlen_enc = vector_length_encoding(this, $mask);
9020 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
9021 $dst$$Register, mask_len, mask_size, vlen_enc);
9022 %}
9023 ins_pipe( pipe_slow );
9024 %}
9025
9026 instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
9027 predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL);
9028 match(Set dst (VectorMaskToLong mask));
9029 format %{ "vector_tolong_bool $dst, $mask \t! using $xtmp as TEMP" %}
9030 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
9031 ins_encode %{
9032 int opcode = this->ideal_Opcode();
9033 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9034 int mask_len = Matcher::vector_length(this, $mask);
9035 int vlen_enc = vector_length_encoding(this, $mask);
9036 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9037 $dst$$Register, mask_len, mbt, vlen_enc);
9038 %}
9039 ins_pipe( pipe_slow );
9040 %}
9041
9042 instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr) %{
9043 predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == NULL);
9044 match(Set dst (VectorMaskToLong (VectorStoreMask mask size)));
9045 format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
9046 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
9047 ins_encode %{
9048 int opcode = this->ideal_Opcode();
9049 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9050 int mask_len = Matcher::vector_length(this, $mask);
9051 int vlen_enc = vector_length_encoding(this, $mask);
9052 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9053 $dst$$Register, mask_len, mbt, vlen_enc);
9054 %}
9055 ins_pipe( pipe_slow );
9056 %}
9057
9058 instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
9059 predicate(n->in(1)->bottom_type()->isa_vectmask());
9060 match(Set dst (VectorMaskTrueCount mask));
9061 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
9062 format %{ "vector_truecount_evex $dst, $mask \t! using $tmp as TEMP" %}
9063 ins_encode %{
9064 int opcode = this->ideal_Opcode();
9065 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9066 int mask_len = Matcher::vector_length(this, $mask);
9067 int mask_size = mask_len * type2aelembytes(mbt);
9068 int vlen_enc = vector_length_encoding(this, $mask);
9069 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
9070 $tmp$$Register, mask_len, mask_size, vlen_enc);
9071 %}
9072 ins_pipe( pipe_slow );
9073 %}
9074
9075 instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
9076 predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL);
9077 match(Set dst (VectorMaskTrueCount mask));
9078 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
9079 format %{ "vector_truecount_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
9080 ins_encode %{
9081 int opcode = this->ideal_Opcode();
9082 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9083 int mask_len = Matcher::vector_length(this, $mask);
9084 int vlen_enc = vector_length_encoding(this, $mask);
9085 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9086 $tmp$$Register, mask_len, mbt, vlen_enc);
9087 %}
9088 ins_pipe( pipe_slow );
9089 %}
9090
9091 instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
9092 predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == NULL);
9093 match(Set dst (VectorMaskTrueCount (VectorStoreMask mask size)));
9094 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
9095 format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
9096 ins_encode %{
9097 int opcode = this->ideal_Opcode();
9098 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9099 int mask_len = Matcher::vector_length(this, $mask);
9100 int vlen_enc = vector_length_encoding(this, $mask);
9101 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9102 $tmp$$Register, mask_len, mbt, vlen_enc);
9103 %}
9104 ins_pipe( pipe_slow );
9105 %}
9106
9107 instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
9108 predicate(n->in(1)->bottom_type()->isa_vectmask());
9109 match(Set dst (VectorMaskFirstTrue mask));
9110 match(Set dst (VectorMaskLastTrue mask));
9111 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
9112 format %{ "vector_mask_first_or_last_true_evex $dst, $mask \t! using $tmp as TEMP" %}
9113 ins_encode %{
9114 int opcode = this->ideal_Opcode();
9115 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9116 int mask_len = Matcher::vector_length(this, $mask);
9117 int mask_size = mask_len * type2aelembytes(mbt);
9118 int vlen_enc = vector_length_encoding(this, $mask);
9119 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
9120 $tmp$$Register, mask_len, mask_size, vlen_enc);
9121 %}
9122 ins_pipe( pipe_slow );
9123 %}
9124
9125 instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
9126 predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL);
9127 match(Set dst (VectorMaskFirstTrue mask));
9128 match(Set dst (VectorMaskLastTrue mask));
9129 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
9130 format %{ "vector_mask_first_or_last_true_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
9131 ins_encode %{
9132 int opcode = this->ideal_Opcode();
9133 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9134 int mask_len = Matcher::vector_length(this, $mask);
9135 int vlen_enc = vector_length_encoding(this, $mask);
9136 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9137 $tmp$$Register, mask_len, mbt, vlen_enc);
9138 %}
9139 ins_pipe( pipe_slow );
9140 %}
9141
9142 instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
9143 predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == NULL);
9144 match(Set dst (VectorMaskFirstTrue (VectorStoreMask mask size)));
9145 match(Set dst (VectorMaskLastTrue (VectorStoreMask mask size)));
9146 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
9147 format %{ "vector_mask_first_or_last_true_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
9148 ins_encode %{
9149 int opcode = this->ideal_Opcode();
9150 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9151 int mask_len = Matcher::vector_length(this, $mask);
9152 int vlen_enc = vector_length_encoding(this, $mask);
9153 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9154 $tmp$$Register, mask_len, mbt, vlen_enc);
9155 %}
9156 ins_pipe( pipe_slow );
9157 %}
9158
9159 // --------------------------------- Compress/Expand Operations ---------------------------
9160
9161 instruct vcompress_expand_reg_evex(vec dst, vec src, kReg mask) %{
9162 match(Set dst (CompressV src mask));
9163 match(Set dst (ExpandV src mask));
9958 uint masklen = Matcher::vector_length(this);
9959 __ knot(masklen, $dst$$KRegister, $src$$KRegister, $ktmp$$KRegister, $rtmp$$Register);
9960 %}
9961 ins_pipe( pipe_slow );
9962 %}
9963
9964 instruct mask_not_imm(kReg dst, kReg src, immI_M1 cnt) %{
9965 predicate((Matcher::vector_length(n) == 8 && VM_Version::supports_avx512dq()) ||
9966 (Matcher::vector_length(n) == 16) ||
9967 (Matcher::vector_length(n) > 16 && VM_Version::supports_avx512bw()));
9968 match(Set dst (XorVMask src (MaskAll cnt)));
9969 format %{ "mask_not $dst, $src, $cnt \t! mask not operation" %}
9970 ins_encode %{
9971 uint masklen = Matcher::vector_length(this);
9972 __ knot(masklen, $dst$$KRegister, $src$$KRegister);
9973 %}
9974 ins_pipe( pipe_slow );
9975 %}
9976
9977 instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp) %{
9978 predicate(n->bottom_type()->isa_vectmask() == NULL && Matcher::vector_length(n) <= 8);
9979 match(Set dst (VectorLongToMask src));
9980 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp);
9981 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp as TEMP" %}
9982 ins_encode %{
9983 int mask_len = Matcher::vector_length(this);
9984 int vec_enc = vector_length_encoding(mask_len);
9985 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
9986 $rtmp2$$Register, xnoreg, mask_len, vec_enc);
9987 %}
9988 ins_pipe( pipe_slow );
9989 %}
9990
9991
9992 instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp1, rFlagsReg cr) %{
9993 predicate(n->bottom_type()->isa_vectmask() == NULL && Matcher::vector_length(n) > 8);
9994 match(Set dst (VectorLongToMask src));
9995 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp1, KILL cr);
9996 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp1, as TEMP" %}
9997 ins_encode %{
9998 int mask_len = Matcher::vector_length(this);
9999 assert(mask_len <= 32, "invalid mask length");
10000 int vec_enc = vector_length_encoding(mask_len);
10001 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
10002 $rtmp2$$Register, $xtmp1$$XMMRegister, mask_len, vec_enc);
10003 %}
10004 ins_pipe( pipe_slow );
10005 %}
10006
10007 instruct long_to_mask_evex(kReg dst, rRegL src) %{
10008 predicate(n->bottom_type()->isa_vectmask());
10009 match(Set dst (VectorLongToMask src));
10010 format %{ "long_to_mask_evex $dst, $src\t!" %}
10011 ins_encode %{
10012 __ kmov($dst$$KRegister, $src$$Register);
10013 %}
|
1285 int MachNode::compute_padding(int current_offset) const {
1286 if (flags() & Node::PD::Flag_intel_jcc_erratum) {
1287 Compile* C = Compile::current();
1288 PhaseOutput* output = C->output();
1289 Block* block = output->block();
1290 int index = output->index();
1291 return IntelJccErratum::compute_padding(current_offset, this, block, index, C->regalloc());
1292 } else {
1293 return 0;
1294 }
1295 }
1296
1297 // Emit exception handler code.
1298 // Stuff framesize into a register and call a VM stub routine.
1299 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
1300
1301 // Note that the code buffer's insts_mark is always relative to insts.
1302 // That's why we must use the macroassembler to generate a handler.
1303 C2_MacroAssembler _masm(&cbuf);
1304 address base = __ start_a_stub(size_exception_handler());
1305 if (base == nullptr) {
1306 ciEnv::current()->record_failure("CodeCache is full");
1307 return 0; // CodeBuffer::expand failed
1308 }
1309 int offset = __ offset();
1310 __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1311 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1312 __ end_a_stub();
1313 return offset;
1314 }
1315
1316 // Emit deopt handler code.
1317 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
1318
1319 // Note that the code buffer's insts_mark is always relative to insts.
1320 // That's why we must use the macroassembler to generate a handler.
1321 C2_MacroAssembler _masm(&cbuf);
1322 address base = __ start_a_stub(size_deopt_handler());
1323 if (base == nullptr) {
1324 ciEnv::current()->record_failure("CodeCache is full");
1325 return 0; // CodeBuffer::expand failed
1326 }
1327 int offset = __ offset();
1328
1329 #ifdef _LP64
1330 address the_pc = (address) __ pc();
1331 Label next;
1332 // push a "the_pc" on the stack without destroying any registers
1333 // as they all may be live.
1334
1335 // push address of "next"
1336 __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
1337 __ bind(next);
1338 // adjust it so it matches "the_pc"
1339 __ subptr(Address(rsp, 0), __ offset() - offset);
1340 #else
1341 InternalAddress here(__ pc());
1342 __ pushptr(here.addr(), noreg);
1343 #endif
2160 return new legVecZOper();
2161 }
2162 if (legacy) {
2163 switch (ideal_reg) {
2164 case Op_VecS: return new legVecSOper();
2165 case Op_VecD: return new legVecDOper();
2166 case Op_VecX: return new legVecXOper();
2167 case Op_VecY: return new legVecYOper();
2168 case Op_VecZ: return new legVecZOper();
2169 }
2170 } else {
2171 switch (ideal_reg) {
2172 case Op_VecS: return new vecSOper();
2173 case Op_VecD: return new vecDOper();
2174 case Op_VecX: return new vecXOper();
2175 case Op_VecY: return new vecYOper();
2176 case Op_VecZ: return new vecZOper();
2177 }
2178 }
2179 ShouldNotReachHere();
2180 return nullptr;
2181 }
2182
2183 bool Matcher::is_reg2reg_move(MachNode* m) {
2184 switch (m->rule()) {
2185 case MoveVec2Leg_rule:
2186 case MoveLeg2Vec_rule:
2187 case MoveF2VL_rule:
2188 case MoveF2LEG_rule:
2189 case MoveVL2F_rule:
2190 case MoveLEG2F_rule:
2191 case MoveD2VL_rule:
2192 case MoveD2LEG_rule:
2193 case MoveVL2D_rule:
2194 case MoveLEG2D_rule:
2195 return true;
2196 default:
2197 return false;
2198 }
2199 }
2200
2329 }
2330 return false;
2331 }
2332
2333 // This function identifies sub-graphs in which a 'load' node is
2334 // input to two different nodes, and such that it can be matched
2335 // with BMI instructions like blsi, blsr, etc.
2336 // Example : for b = -a[i] & a[i] can be matched to blsi r32, m32.
2337 // The graph is (AndL (SubL Con0 LoadL*) LoadL*), where LoadL*
2338 // refers to the same node.
2339 //
2340 // Match the generic fused operations pattern (op1 (op2 Con{ConType} mop) mop)
2341 // This is a temporary solution until we make DAGs expressible in ADL.
2342 template<typename ConType>
2343 class FusedPatternMatcher {
2344 Node* _op1_node;
2345 Node* _mop_node;
2346 int _con_op;
2347
2348 static int match_next(Node* n, int next_op, int next_op_idx) {
2349 if (n->in(1) == nullptr || n->in(2) == nullptr) {
2350 return -1;
2351 }
2352
2353 if (next_op_idx == -1) { // n is commutative, try rotations
2354 if (n->in(1)->Opcode() == next_op) {
2355 return 1;
2356 } else if (n->in(2)->Opcode() == next_op) {
2357 return 2;
2358 }
2359 } else {
2360 assert(next_op_idx > 0 && next_op_idx <= 2, "Bad argument index");
2361 if (n->in(next_op_idx)->Opcode() == next_op) {
2362 return next_op_idx;
2363 }
2364 }
2365 return -1;
2366 }
2367
2368 public:
2369 FusedPatternMatcher(Node* op1_node, Node* mop_node, int con_op) :
2396 if (op2_con_idx == -1) {
2397 return false;
2398 }
2399 // Memory operation must be the other edge
2400 int op2_mop_idx = (op2_con_idx & 1) + 1;
2401 // Check that the memory operation is the same node
2402 if (op2_node->in(op2_mop_idx) == _mop_node) {
2403 // Now check the constant
2404 const Type* con_type = op2_node->in(op2_con_idx)->bottom_type();
2405 if (con_type != Type::TOP && ConType::as_self(con_type)->get_con() == con_value) {
2406 return true;
2407 }
2408 }
2409 }
2410 return false;
2411 }
2412 };
2413
2414 static bool is_bmi_pattern(Node* n, Node* m) {
2415 assert(UseBMI1Instructions, "sanity");
2416 if (n != nullptr && m != nullptr) {
2417 if (m->Opcode() == Op_LoadI) {
2418 FusedPatternMatcher<TypeInt> bmii(n, m, Op_ConI);
2419 return bmii.match(Op_AndI, -1, Op_SubI, 1, 0) ||
2420 bmii.match(Op_AndI, -1, Op_AddI, -1, -1) ||
2421 bmii.match(Op_XorI, -1, Op_AddI, -1, -1);
2422 } else if (m->Opcode() == Op_LoadL) {
2423 FusedPatternMatcher<TypeLong> bmil(n, m, Op_ConL);
2424 return bmil.match(Op_AndL, -1, Op_SubL, 1, 0) ||
2425 bmil.match(Op_AndL, -1, Op_AddL, -1, -1) ||
2426 bmil.match(Op_XorL, -1, Op_AddL, -1, -1);
2427 }
2428 }
2429 return false;
2430 }
2431
2432 // Should the matcher clone input 'm' of node 'n'?
2433 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2434 // If 'n' and 'm' are part of a graph for BMI instruction, clone the input 'm'.
2435 if (UseBMI1Instructions && is_bmi_pattern(n, m)) {
2436 mstack.push(m, Visit);
2762 uint MachBreakpointNode::size(PhaseRegAlloc* ra_) const {
2763 return MachNode::size(ra_);
2764 }
2765
2766 %}
2767
2768 encode %{
2769
2770 enc_class call_epilog %{
2771 C2_MacroAssembler _masm(&cbuf);
2772 if (VerifyStackAtCalls) {
2773 // Check that stack depth is unchanged: find majik cookie on stack
2774 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
2775 Label L;
2776 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
2777 __ jccb(Assembler::equal, L);
2778 // Die if stack mismatch
2779 __ int3();
2780 __ bind(L);
2781 }
2782 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
2783 C2_MacroAssembler _masm(&cbuf);
2784 if (!_method->signature()->returns_null_free_inline_type()) {
2785 // The last return value is not set by the callee but used to pass IsInit information to compiled code.
2786 // Search for the corresponding projection, get the register and emit code that initialized it.
2787 uint con = (tf()->range_cc()->cnt() - 1);
2788 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
2789 ProjNode* proj = fast_out(i)->as_Proj();
2790 if (proj->_con == con) {
2791 // Set IsInit if rax is non-null (a non-null value is returned buffered or scalarized)
2792 OptoReg::Name optoReg = ra_->get_reg_first(proj);
2793 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
2794 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
2795 __ testq(rax, rax);
2796 __ setb(Assembler::notZero, toReg);
2797 __ movzbl(toReg, toReg);
2798 if (reg->is_stack()) {
2799 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
2800 __ movq(Address(rsp, st_off), toReg);
2801 }
2802 break;
2803 }
2804 }
2805 }
2806 if (return_value_is_used()) {
2807 // An inline type is returned as fields in multiple registers.
2808 // Rax either contains an oop if the inline type is buffered or a pointer
2809 // to the corresponding InlineKlass with the lowest bit set to 1. Zero rax
2810 // if the lowest bit is set to allow C2 to use the oop after null checking.
2811 // rax &= (rax & 1) - 1
2812 __ movptr(rscratch1, rax);
2813 __ andptr(rscratch1, 0x1);
2814 __ subptr(rscratch1, 0x1);
2815 __ andptr(rax, rscratch1);
2816 }
2817 }
2818 %}
2819
2820 %}
2821
2822 // Operands for bound floating pointer register arguments
2823 operand rxmm0() %{
2824 constraint(ALLOC_IN_RC(xmm0_reg));
2825 match(VecX);
2826 format%{%}
2827 interface(REG_INTER);
2828 %}
2829
2830 //----------OPERANDS-----------------------------------------------------------
2831 // Operand definitions must precede instruction definitions for correct parsing
2832 // in the ADLC because operands constitute user defined types which are used in
2833 // instruction definitions.
2834
2835 // Vectors
2836
2837 // Dummy generic vector class. Should be used for all vector operands.
7475 instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
7476 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
7477 match(Set dst (RoundVD src));
7478 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
7479 format %{ "vector_round_long $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
7480 ins_encode %{
7481 int vlen_enc = vector_length_encoding(this);
7482 InternalAddress new_mxcsr = $constantaddress((jint)0x3F80);
7483 __ vector_round_double_evex($dst$$XMMRegister, $src$$XMMRegister,
7484 ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), new_mxcsr, vlen_enc,
7485 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
7486 %}
7487 ins_pipe( pipe_slow );
7488 %}
7489
7490 #endif // _LP64
7491
7492 // --------------------------------- VectorMaskCmp --------------------------------------
7493
7494 instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
7495 predicate(n->bottom_type()->isa_vectmask() == nullptr &&
7496 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 8 && // src1
7497 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
7498 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
7499 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7500 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
7501 ins_encode %{
7502 int vlen_enc = vector_length_encoding(this, $src1);
7503 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
7504 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
7505 __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7506 } else {
7507 __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7508 }
7509 %}
7510 ins_pipe( pipe_slow );
7511 %}
7512
7513 instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
7514 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
7515 n->bottom_type()->isa_vectmask() == nullptr &&
7516 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
7517 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7518 effect(TEMP ktmp);
7519 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
7520 ins_encode %{
7521 int vlen_enc = Assembler::AVX_512bit;
7522 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
7523 KRegister mask = k0; // The comparison itself is not being masked.
7524 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
7525 __ evcmpps($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7526 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
7527 } else {
7528 __ evcmppd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7529 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
7530 }
7531 %}
7532 ins_pipe( pipe_slow );
7533 %}
7534
7535 instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
7536 predicate(n->bottom_type()->isa_vectmask() &&
7537 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
7538 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7539 format %{ "vector_compare_evex $dst,$src1,$src2,$cond\t!" %}
7540 ins_encode %{
7541 assert(bottom_type()->isa_vectmask(), "TypeVectMask expected");
7542 int vlen_enc = vector_length_encoding(this, $src1);
7543 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
7544 KRegister mask = k0; // The comparison itself is not being masked.
7545 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
7546 __ evcmpps($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7547 } else {
7548 __ evcmppd($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
7549 }
7550 %}
7551 ins_pipe( pipe_slow );
7552 %}
7553
7554 instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
7555 predicate(n->bottom_type()->isa_vectmask() == nullptr &&
7556 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
7557 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
7558 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
7559 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
7560 (n->in(2)->get_int() == BoolTest::eq ||
7561 n->in(2)->get_int() == BoolTest::lt ||
7562 n->in(2)->get_int() == BoolTest::gt)); // cond
7563 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7564 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
7565 ins_encode %{
7566 int vlen_enc = vector_length_encoding(this, $src1);
7567 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
7568 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
7569 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, cmp, ww, vlen_enc);
7570 %}
7571 ins_pipe( pipe_slow );
7572 %}
7573
7574 instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
7575 predicate(n->bottom_type()->isa_vectmask() == nullptr &&
7576 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
7577 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
7578 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
7579 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
7580 (n->in(2)->get_int() == BoolTest::ne ||
7581 n->in(2)->get_int() == BoolTest::le ||
7582 n->in(2)->get_int() == BoolTest::ge)); // cond
7583 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7584 effect(TEMP dst, TEMP xtmp);
7585 format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
7586 ins_encode %{
7587 int vlen_enc = vector_length_encoding(this, $src1);
7588 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
7589 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
7590 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
7591 %}
7592 ins_pipe( pipe_slow );
7593 %}
7594
7595 instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
7596 predicate(n->bottom_type()->isa_vectmask() == nullptr &&
7597 Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
7598 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
7599 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
7600 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
7601 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7602 effect(TEMP dst, TEMP xtmp);
7603 format %{ "vector_compareu $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
7604 ins_encode %{
7605 InternalAddress flip_bit = $constantaddress(high_bit_set(Matcher::vector_element_basic_type(this, $src1)));
7606 int vlen_enc = vector_length_encoding(this, $src1);
7607 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
7608 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
7609
7610 if (vlen_enc == Assembler::AVX_128bit) {
7611 __ vmovddup($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
7612 } else {
7613 __ vbroadcastsd($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
7614 }
7615 __ vpxor($dst$$XMMRegister, $xtmp$$XMMRegister, $src1$$XMMRegister, vlen_enc);
7616 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $src2$$XMMRegister, vlen_enc);
7617 __ vpcmpCCW($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
7618 %}
7619 ins_pipe( pipe_slow );
7620 %}
7621
7622 instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
7623 predicate((n->bottom_type()->isa_vectmask() == nullptr &&
7624 Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
7625 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
7626 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
7627 effect(TEMP ktmp);
7628 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
7629 ins_encode %{
7630 assert(UseAVX > 2, "required");
7631
7632 int vlen_enc = vector_length_encoding(this, $src1);
7633 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
7634 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
7635 KRegister mask = k0; // The comparison itself is not being masked.
7636 bool merge = false;
7637 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
7638
7639 switch (src1_elem_bt) {
7640 case T_INT: {
7641 __ evpcmpd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
7642 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
7643 break;
7819 // --------------------------------- Vector Blend --------------------------------------
7820
7821 instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
7822 predicate(UseAVX == 0);
7823 match(Set dst (VectorBlend (Binary dst src) mask));
7824 format %{ "vector_blend $dst,$src,$mask\t! using $tmp as TEMP" %}
7825 effect(TEMP tmp);
7826 ins_encode %{
7827 assert(UseSSE >= 4, "required");
7828
7829 if ($mask$$XMMRegister != $tmp$$XMMRegister) {
7830 __ movdqu($tmp$$XMMRegister, $mask$$XMMRegister);
7831 }
7832 __ pblendvb($dst$$XMMRegister, $src$$XMMRegister); // uses xmm0 as mask
7833 %}
7834 ins_pipe( pipe_slow );
7835 %}
7836
7837 instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
7838 predicate(UseAVX > 0 &&
7839 n->in(2)->bottom_type()->isa_vectmask() == nullptr &&
7840 Matcher::vector_length_in_bytes(n) <= 32 &&
7841 is_integral_type(Matcher::vector_element_basic_type(n)));
7842 match(Set dst (VectorBlend (Binary src1 src2) mask));
7843 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
7844 ins_encode %{
7845 int vlen_enc = vector_length_encoding(this);
7846 __ vpblendvb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
7847 %}
7848 ins_pipe( pipe_slow );
7849 %}
7850
7851 instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
7852 predicate(UseAVX > 0 &&
7853 n->in(2)->bottom_type()->isa_vectmask() == nullptr &&
7854 Matcher::vector_length_in_bytes(n) <= 32 &&
7855 !is_integral_type(Matcher::vector_element_basic_type(n)));
7856 match(Set dst (VectorBlend (Binary src1 src2) mask));
7857 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
7858 ins_encode %{
7859 int vlen_enc = vector_length_encoding(this);
7860 __ vblendvps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
7861 %}
7862 ins_pipe( pipe_slow );
7863 %}
7864
7865 instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
7866 predicate(Matcher::vector_length_in_bytes(n) == 64 &&
7867 n->in(2)->bottom_type()->isa_vectmask() == nullptr);
7868 match(Set dst (VectorBlend (Binary src1 src2) mask));
7869 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
7870 effect(TEMP ktmp);
7871 ins_encode %{
7872 int vlen_enc = Assembler::AVX_512bit;
7873 BasicType elem_bt = Matcher::vector_element_basic_type(this);
7874 __ evpcmp(elem_bt, $ktmp$$KRegister, k0, $mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), Assembler::eq, vlen_enc, noreg);
7875 __ evpblend(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
7876 %}
7877 ins_pipe( pipe_slow );
7878 %}
7879
7880
7881 instruct evblendvp64_masked(vec dst, vec src1, vec src2, kReg mask) %{
7882 predicate(n->in(2)->bottom_type()->isa_vectmask() &&
7883 (!is_subword_type(Matcher::vector_element_basic_type(n)) ||
7884 VM_Version::supports_avx512bw()));
7885 match(Set dst (VectorBlend (Binary src1 src2) mask));
7886 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
7887 ins_encode %{
8064 %}
8065 ins_pipe( pipe_slow );
8066 %}
8067
8068 instruct ktest_ge8(rFlagsRegU cr, kReg src1, kReg src2) %{
8069 predicate(Matcher::vector_length(n->in(1)) >= 16 ||
8070 (Matcher::vector_length(n->in(1)) == 8 && VM_Version::supports_avx512dq()));
8071 match(Set cr (VectorTest src1 src2));
8072 format %{ "ktest_ge8 $src1, $src2\n\t" %}
8073 ins_encode %{
8074 uint masklen = Matcher::vector_length(this, $src1);
8075 __ kortest(masklen, $src1$$KRegister, $src1$$KRegister);
8076 %}
8077 ins_pipe( pipe_slow );
8078 %}
8079 #endif
8080
8081 //------------------------------------- LoadMask --------------------------------------------
8082
8083 instruct loadMask(legVec dst, legVec src) %{
8084 predicate(n->bottom_type()->isa_vectmask() == nullptr && !VM_Version::supports_avx512vlbw());
8085 match(Set dst (VectorLoadMask src));
8086 effect(TEMP dst);
8087 format %{ "vector_loadmask_byte $dst, $src\n\t" %}
8088 ins_encode %{
8089 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
8090 BasicType elem_bt = Matcher::vector_element_basic_type(this);
8091 __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt, true);
8092 %}
8093 ins_pipe( pipe_slow );
8094 %}
8095
8096 instruct loadMask64(kReg dst, vec src, vec xtmp) %{
8097 predicate(n->bottom_type()->isa_vectmask() && !VM_Version::supports_avx512vlbw());
8098 match(Set dst (VectorLoadMask src));
8099 effect(TEMP xtmp);
8100 format %{ "vector_loadmask_64byte $dst, $src\t! using $xtmp as TEMP" %}
8101 ins_encode %{
8102 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
8103 true, Assembler::AVX_512bit);
8104 %}
8105 ins_pipe( pipe_slow );
8106 %}
8107
8108 instruct loadMask_evex(kReg dst, vec src, vec xtmp) %{
8109 predicate(n->bottom_type()->isa_vectmask() && VM_Version::supports_avx512vlbw());
8110 match(Set dst (VectorLoadMask src));
8111 effect(TEMP xtmp);
8112 format %{ "vector_loadmask_byte $dst, $src\t! using $xtmp as TEMP" %}
8113 ins_encode %{
8114 int vlen_enc = vector_length_encoding(in(1));
8115 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
8116 false, vlen_enc);
8117 %}
8118 ins_pipe( pipe_slow );
8119 %}
8120
8121 //------------------------------------- StoreMask --------------------------------------------
8122
8123 instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
8124 predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
8125 match(Set dst (VectorStoreMask src size));
8126 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
8127 ins_encode %{
8128 int vlen = Matcher::vector_length(this);
8129 if (vlen <= 16 && UseAVX <= 2) {
8130 assert(UseSSE >= 3, "required");
8131 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
8132 } else {
8133 assert(UseAVX > 0, "required");
8134 int src_vlen_enc = vector_length_encoding(this, $src);
8135 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
8136 }
8137 %}
8138 ins_pipe( pipe_slow );
8139 %}
8140
8141 instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
8142 predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
8143 match(Set dst (VectorStoreMask src size));
8144 effect(TEMP_DEF dst, TEMP xtmp);
8145 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
8146 ins_encode %{
8147 int vlen_enc = Assembler::AVX_128bit;
8148 int vlen = Matcher::vector_length(this);
8149 if (vlen <= 8) {
8150 assert(UseSSE >= 3, "required");
8151 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
8152 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
8153 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
8154 } else {
8155 assert(UseAVX > 0, "required");
8156 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
8157 __ vpacksswb($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
8158 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
8159 }
8160 %}
8161 ins_pipe( pipe_slow );
8162 %}
8163
8164 instruct vstoreMask4B(vec dst, vec src, vec xtmp, immI_4 size) %{
8165 predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
8166 match(Set dst (VectorStoreMask src size));
8167 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
8168 effect(TEMP_DEF dst, TEMP xtmp);
8169 ins_encode %{
8170 int vlen_enc = Assembler::AVX_128bit;
8171 int vlen = Matcher::vector_length(this);
8172 if (vlen <= 4) {
8173 assert(UseSSE >= 3, "required");
8174 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
8175 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
8176 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
8177 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
8178 } else {
8179 assert(UseAVX > 0, "required");
8180 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
8181 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
8182 __ vpackssdw($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
8183 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
8184 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
8185 }
8205
8206 instruct storeMask8B_avx(vec dst, vec src, immI_8 size, vec vtmp) %{
8207 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 4);
8208 match(Set dst (VectorStoreMask src size));
8209 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s], using $vtmp as TEMP" %}
8210 effect(TEMP_DEF dst, TEMP vtmp);
8211 ins_encode %{
8212 int vlen_enc = Assembler::AVX_128bit;
8213 __ vshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 0x88, Assembler::AVX_256bit);
8214 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
8215 __ vblendps($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0xC, vlen_enc);
8216 __ vpxor($vtmp$$XMMRegister, $vtmp$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
8217 __ vpackssdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
8218 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
8219 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
8220 %}
8221 ins_pipe( pipe_slow );
8222 %}
8223
8224 instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
8225 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
8226 match(Set dst (VectorStoreMask src size));
8227 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
8228 ins_encode %{
8229 int src_vlen_enc = vector_length_encoding(this, $src);
8230 int dst_vlen_enc = vector_length_encoding(this);
8231 if (!VM_Version::supports_avx512vl()) {
8232 src_vlen_enc = Assembler::AVX_512bit;
8233 }
8234 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
8235 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
8236 %}
8237 ins_pipe( pipe_slow );
8238 %}
8239
8240 instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
8241 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
8242 match(Set dst (VectorStoreMask src size));
8243 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
8244 ins_encode %{
8245 int src_vlen_enc = vector_length_encoding(this, $src);
8246 int dst_vlen_enc = vector_length_encoding(this);
8247 if (!VM_Version::supports_avx512vl()) {
8248 src_vlen_enc = Assembler::AVX_512bit;
8249 }
8250 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
8251 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
8252 %}
8253 ins_pipe( pipe_slow );
8254 %}
8255
8256 instruct vstoreMask_evex_vectmask(vec dst, kReg mask, immI size) %{
8257 predicate(n->in(1)->bottom_type()->isa_vectmask() && !VM_Version::supports_avx512vlbw());
8258 match(Set dst (VectorStoreMask mask size));
8259 effect(TEMP_DEF dst);
8260 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
8261 ins_encode %{
9043 %}
9044
9045 instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
9046 predicate(n->in(1)->bottom_type()->isa_vectmask());
9047 match(Set dst (VectorMaskToLong mask));
9048 effect(TEMP dst, KILL cr);
9049 format %{ "vector_tolong_evex $dst, $mask \t! vector mask tolong" %}
9050 ins_encode %{
9051 int opcode = this->ideal_Opcode();
9052 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9053 int mask_len = Matcher::vector_length(this, $mask);
9054 int mask_size = mask_len * type2aelembytes(mbt);
9055 int vlen_enc = vector_length_encoding(this, $mask);
9056 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
9057 $dst$$Register, mask_len, mask_size, vlen_enc);
9058 %}
9059 ins_pipe( pipe_slow );
9060 %}
9061
9062 instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
9063 predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
9064 match(Set dst (VectorMaskToLong mask));
9065 format %{ "vector_tolong_bool $dst, $mask \t! using $xtmp as TEMP" %}
9066 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
9067 ins_encode %{
9068 int opcode = this->ideal_Opcode();
9069 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9070 int mask_len = Matcher::vector_length(this, $mask);
9071 int vlen_enc = vector_length_encoding(this, $mask);
9072 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9073 $dst$$Register, mask_len, mbt, vlen_enc);
9074 %}
9075 ins_pipe( pipe_slow );
9076 %}
9077
9078 instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr) %{
9079 predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
9080 match(Set dst (VectorMaskToLong (VectorStoreMask mask size)));
9081 format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
9082 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
9083 ins_encode %{
9084 int opcode = this->ideal_Opcode();
9085 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9086 int mask_len = Matcher::vector_length(this, $mask);
9087 int vlen_enc = vector_length_encoding(this, $mask);
9088 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9089 $dst$$Register, mask_len, mbt, vlen_enc);
9090 %}
9091 ins_pipe( pipe_slow );
9092 %}
9093
9094 instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
9095 predicate(n->in(1)->bottom_type()->isa_vectmask());
9096 match(Set dst (VectorMaskTrueCount mask));
9097 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
9098 format %{ "vector_truecount_evex $dst, $mask \t! using $tmp as TEMP" %}
9099 ins_encode %{
9100 int opcode = this->ideal_Opcode();
9101 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9102 int mask_len = Matcher::vector_length(this, $mask);
9103 int mask_size = mask_len * type2aelembytes(mbt);
9104 int vlen_enc = vector_length_encoding(this, $mask);
9105 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
9106 $tmp$$Register, mask_len, mask_size, vlen_enc);
9107 %}
9108 ins_pipe( pipe_slow );
9109 %}
9110
9111 instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
9112 predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
9113 match(Set dst (VectorMaskTrueCount mask));
9114 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
9115 format %{ "vector_truecount_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
9116 ins_encode %{
9117 int opcode = this->ideal_Opcode();
9118 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9119 int mask_len = Matcher::vector_length(this, $mask);
9120 int vlen_enc = vector_length_encoding(this, $mask);
9121 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9122 $tmp$$Register, mask_len, mbt, vlen_enc);
9123 %}
9124 ins_pipe( pipe_slow );
9125 %}
9126
9127 instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
9128 predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
9129 match(Set dst (VectorMaskTrueCount (VectorStoreMask mask size)));
9130 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
9131 format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
9132 ins_encode %{
9133 int opcode = this->ideal_Opcode();
9134 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9135 int mask_len = Matcher::vector_length(this, $mask);
9136 int vlen_enc = vector_length_encoding(this, $mask);
9137 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9138 $tmp$$Register, mask_len, mbt, vlen_enc);
9139 %}
9140 ins_pipe( pipe_slow );
9141 %}
9142
9143 instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
9144 predicate(n->in(1)->bottom_type()->isa_vectmask());
9145 match(Set dst (VectorMaskFirstTrue mask));
9146 match(Set dst (VectorMaskLastTrue mask));
9147 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
9148 format %{ "vector_mask_first_or_last_true_evex $dst, $mask \t! using $tmp as TEMP" %}
9149 ins_encode %{
9150 int opcode = this->ideal_Opcode();
9151 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9152 int mask_len = Matcher::vector_length(this, $mask);
9153 int mask_size = mask_len * type2aelembytes(mbt);
9154 int vlen_enc = vector_length_encoding(this, $mask);
9155 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
9156 $tmp$$Register, mask_len, mask_size, vlen_enc);
9157 %}
9158 ins_pipe( pipe_slow );
9159 %}
9160
9161 instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
9162 predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
9163 match(Set dst (VectorMaskFirstTrue mask));
9164 match(Set dst (VectorMaskLastTrue mask));
9165 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
9166 format %{ "vector_mask_first_or_last_true_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
9167 ins_encode %{
9168 int opcode = this->ideal_Opcode();
9169 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9170 int mask_len = Matcher::vector_length(this, $mask);
9171 int vlen_enc = vector_length_encoding(this, $mask);
9172 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9173 $tmp$$Register, mask_len, mbt, vlen_enc);
9174 %}
9175 ins_pipe( pipe_slow );
9176 %}
9177
9178 instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
9179 predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
9180 match(Set dst (VectorMaskFirstTrue (VectorStoreMask mask size)));
9181 match(Set dst (VectorMaskLastTrue (VectorStoreMask mask size)));
9182 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
9183 format %{ "vector_mask_first_or_last_true_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
9184 ins_encode %{
9185 int opcode = this->ideal_Opcode();
9186 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
9187 int mask_len = Matcher::vector_length(this, $mask);
9188 int vlen_enc = vector_length_encoding(this, $mask);
9189 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
9190 $tmp$$Register, mask_len, mbt, vlen_enc);
9191 %}
9192 ins_pipe( pipe_slow );
9193 %}
9194
9195 // --------------------------------- Compress/Expand Operations ---------------------------
9196
9197 instruct vcompress_expand_reg_evex(vec dst, vec src, kReg mask) %{
9198 match(Set dst (CompressV src mask));
9199 match(Set dst (ExpandV src mask));
9994 uint masklen = Matcher::vector_length(this);
9995 __ knot(masklen, $dst$$KRegister, $src$$KRegister, $ktmp$$KRegister, $rtmp$$Register);
9996 %}
9997 ins_pipe( pipe_slow );
9998 %}
9999
10000 instruct mask_not_imm(kReg dst, kReg src, immI_M1 cnt) %{
10001 predicate((Matcher::vector_length(n) == 8 && VM_Version::supports_avx512dq()) ||
10002 (Matcher::vector_length(n) == 16) ||
10003 (Matcher::vector_length(n) > 16 && VM_Version::supports_avx512bw()));
10004 match(Set dst (XorVMask src (MaskAll cnt)));
10005 format %{ "mask_not $dst, $src, $cnt \t! mask not operation" %}
10006 ins_encode %{
10007 uint masklen = Matcher::vector_length(this);
10008 __ knot(masklen, $dst$$KRegister, $src$$KRegister);
10009 %}
10010 ins_pipe( pipe_slow );
10011 %}
10012
10013 instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp) %{
10014 predicate(n->bottom_type()->isa_vectmask() == nullptr && Matcher::vector_length(n) <= 8);
10015 match(Set dst (VectorLongToMask src));
10016 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp);
10017 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp as TEMP" %}
10018 ins_encode %{
10019 int mask_len = Matcher::vector_length(this);
10020 int vec_enc = vector_length_encoding(mask_len);
10021 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
10022 $rtmp2$$Register, xnoreg, mask_len, vec_enc);
10023 %}
10024 ins_pipe( pipe_slow );
10025 %}
10026
10027
10028 instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp1, rFlagsReg cr) %{
10029 predicate(n->bottom_type()->isa_vectmask() == nullptr && Matcher::vector_length(n) > 8);
10030 match(Set dst (VectorLongToMask src));
10031 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp1, KILL cr);
10032 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp1, as TEMP" %}
10033 ins_encode %{
10034 int mask_len = Matcher::vector_length(this);
10035 assert(mask_len <= 32, "invalid mask length");
10036 int vec_enc = vector_length_encoding(mask_len);
10037 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
10038 $rtmp2$$Register, $xtmp1$$XMMRegister, mask_len, vec_enc);
10039 %}
10040 ins_pipe( pipe_slow );
10041 %}
10042
10043 instruct long_to_mask_evex(kReg dst, rRegL src) %{
10044 predicate(n->bottom_type()->isa_vectmask());
10045 match(Set dst (VectorLongToMask src));
10046 format %{ "long_to_mask_evex $dst, $src\t!" %}
10047 ins_encode %{
10048 __ kmov($dst$$KRegister, $src$$Register);
10049 %}
|