< prev index next >

src/hotspot/cpu/x86/x86.ad

Print this page

 1285 int MachNode::compute_padding(int current_offset) const {
 1286   if (flags() & Node::PD::Flag_intel_jcc_erratum) {
 1287     Compile* C = Compile::current();
 1288     PhaseOutput* output = C->output();
 1289     Block* block = output->block();
 1290     int index = output->index();
 1291     return IntelJccErratum::compute_padding(current_offset, this, block, index, C->regalloc());
 1292   } else {
 1293     return 0;
 1294   }
 1295 }
 1296 
 1297 // Emit exception handler code.
 1298 // Stuff framesize into a register and call a VM stub routine.
 1299 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
 1300 
 1301   // Note that the code buffer's insts_mark is always relative to insts.
 1302   // That's why we must use the macroassembler to generate a handler.
 1303   C2_MacroAssembler _masm(&cbuf);
 1304   address base = __ start_a_stub(size_exception_handler());
 1305   if (base == NULL) {
 1306     ciEnv::current()->record_failure("CodeCache is full");
 1307     return 0;  // CodeBuffer::expand failed
 1308   }
 1309   int offset = __ offset();
 1310   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 1311   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 1312   __ end_a_stub();
 1313   return offset;
 1314 }
 1315 
 1316 // Emit deopt handler code.
 1317 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
 1318 
 1319   // Note that the code buffer's insts_mark is always relative to insts.
 1320   // That's why we must use the macroassembler to generate a handler.
 1321   C2_MacroAssembler _masm(&cbuf);
 1322   address base = __ start_a_stub(size_deopt_handler());
 1323   if (base == NULL) {
 1324     ciEnv::current()->record_failure("CodeCache is full");
 1325     return 0;  // CodeBuffer::expand failed
 1326   }
 1327   int offset = __ offset();
 1328 
 1329 #ifdef _LP64
 1330   address the_pc = (address) __ pc();
 1331   Label next;
 1332   // push a "the_pc" on the stack without destroying any registers
 1333   // as they all may be live.
 1334 
 1335   // push address of "next"
 1336   __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
 1337   __ bind(next);
 1338   // adjust it so it matches "the_pc"
 1339   __ subptr(Address(rsp, 0), __ offset() - offset);
 1340 #else
 1341   InternalAddress here(__ pc());
 1342   __ pushptr(here.addr(), noreg);
 1343 #endif

 2160     return new legVecZOper();
 2161   }
 2162   if (legacy) {
 2163     switch (ideal_reg) {
 2164       case Op_VecS: return new legVecSOper();
 2165       case Op_VecD: return new legVecDOper();
 2166       case Op_VecX: return new legVecXOper();
 2167       case Op_VecY: return new legVecYOper();
 2168       case Op_VecZ: return new legVecZOper();
 2169     }
 2170   } else {
 2171     switch (ideal_reg) {
 2172       case Op_VecS: return new vecSOper();
 2173       case Op_VecD: return new vecDOper();
 2174       case Op_VecX: return new vecXOper();
 2175       case Op_VecY: return new vecYOper();
 2176       case Op_VecZ: return new vecZOper();
 2177     }
 2178   }
 2179   ShouldNotReachHere();
 2180   return NULL;
 2181 }
 2182 
 2183 bool Matcher::is_reg2reg_move(MachNode* m) {
 2184   switch (m->rule()) {
 2185     case MoveVec2Leg_rule:
 2186     case MoveLeg2Vec_rule:
 2187     case MoveF2VL_rule:
 2188     case MoveF2LEG_rule:
 2189     case MoveVL2F_rule:
 2190     case MoveLEG2F_rule:
 2191     case MoveD2VL_rule:
 2192     case MoveD2LEG_rule:
 2193     case MoveVL2D_rule:
 2194     case MoveLEG2D_rule:
 2195       return true;
 2196     default:
 2197       return false;
 2198   }
 2199 }
 2200 

 2329   }
 2330   return false;
 2331 }
 2332 
 2333 // This function identifies sub-graphs in which a 'load' node is
 2334 // input to two different nodes, and such that it can be matched
 2335 // with BMI instructions like blsi, blsr, etc.
 2336 // Example : for b = -a[i] & a[i] can be matched to blsi r32, m32.
 2337 // The graph is (AndL (SubL Con0 LoadL*) LoadL*), where LoadL*
 2338 // refers to the same node.
 2339 //
 2340 // Match the generic fused operations pattern (op1 (op2 Con{ConType} mop) mop)
 2341 // This is a temporary solution until we make DAGs expressible in ADL.
 2342 template<typename ConType>
 2343 class FusedPatternMatcher {
 2344   Node* _op1_node;
 2345   Node* _mop_node;
 2346   int _con_op;
 2347 
 2348   static int match_next(Node* n, int next_op, int next_op_idx) {
 2349     if (n->in(1) == NULL || n->in(2) == NULL) {
 2350       return -1;
 2351     }
 2352 
 2353     if (next_op_idx == -1) { // n is commutative, try rotations
 2354       if (n->in(1)->Opcode() == next_op) {
 2355         return 1;
 2356       } else if (n->in(2)->Opcode() == next_op) {
 2357         return 2;
 2358       }
 2359     } else {
 2360       assert(next_op_idx > 0 && next_op_idx <= 2, "Bad argument index");
 2361       if (n->in(next_op_idx)->Opcode() == next_op) {
 2362         return next_op_idx;
 2363       }
 2364     }
 2365     return -1;
 2366   }
 2367 
 2368  public:
 2369   FusedPatternMatcher(Node* op1_node, Node* mop_node, int con_op) :

 2396       if (op2_con_idx == -1) {
 2397         return false;
 2398       }
 2399       // Memory operation must be the other edge
 2400       int op2_mop_idx = (op2_con_idx & 1) + 1;
 2401       // Check that the memory operation is the same node
 2402       if (op2_node->in(op2_mop_idx) == _mop_node) {
 2403         // Now check the constant
 2404         const Type* con_type = op2_node->in(op2_con_idx)->bottom_type();
 2405         if (con_type != Type::TOP && ConType::as_self(con_type)->get_con() == con_value) {
 2406           return true;
 2407         }
 2408       }
 2409     }
 2410     return false;
 2411   }
 2412 };
 2413 
 2414 static bool is_bmi_pattern(Node* n, Node* m) {
 2415   assert(UseBMI1Instructions, "sanity");
 2416   if (n != NULL && m != NULL) {
 2417     if (m->Opcode() == Op_LoadI) {
 2418       FusedPatternMatcher<TypeInt> bmii(n, m, Op_ConI);
 2419       return bmii.match(Op_AndI, -1, Op_SubI,  1,  0)  ||
 2420              bmii.match(Op_AndI, -1, Op_AddI, -1, -1)  ||
 2421              bmii.match(Op_XorI, -1, Op_AddI, -1, -1);
 2422     } else if (m->Opcode() == Op_LoadL) {
 2423       FusedPatternMatcher<TypeLong> bmil(n, m, Op_ConL);
 2424       return bmil.match(Op_AndL, -1, Op_SubL,  1,  0) ||
 2425              bmil.match(Op_AndL, -1, Op_AddL, -1, -1) ||
 2426              bmil.match(Op_XorL, -1, Op_AddL, -1, -1);
 2427     }
 2428   }
 2429   return false;
 2430 }
 2431 
 2432 // Should the matcher clone input 'm' of node 'n'?
 2433 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2434   // If 'n' and 'm' are part of a graph for BMI instruction, clone the input 'm'.
 2435   if (UseBMI1Instructions && is_bmi_pattern(n, m)) {
 2436     mstack.push(m, Visit);

 2762   uint MachBreakpointNode::size(PhaseRegAlloc* ra_) const {
 2763     return MachNode::size(ra_);
 2764   }
 2765 
 2766 %}
 2767 
 2768 encode %{
 2769 
 2770   enc_class call_epilog %{
 2771     C2_MacroAssembler _masm(&cbuf);
 2772     if (VerifyStackAtCalls) {
 2773       // Check that stack depth is unchanged: find majik cookie on stack
 2774       int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
 2775       Label L;
 2776       __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
 2777       __ jccb(Assembler::equal, L);
 2778       // Die if stack mismatch
 2779       __ int3();
 2780       __ bind(L);
 2781     }




































 2782   %}
 2783 
 2784 %}
 2785 
 2786 // Operands for bound floating pointer register arguments
 2787 operand rxmm0() %{
 2788   constraint(ALLOC_IN_RC(xmm0_reg));
 2789   match(VecX);
 2790   format%{%}
 2791   interface(REG_INTER);
 2792 %}
 2793 
 2794 //----------OPERANDS-----------------------------------------------------------
 2795 // Operand definitions must precede instruction definitions for correct parsing
 2796 // in the ADLC because operands constitute user defined types which are used in
 2797 // instruction definitions.
 2798 
 2799 // Vectors
 2800 
 2801 // Dummy generic vector class. Should be used for all vector operands.

 7439 instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
 7440   predicate(Matcher::vector_element_basic_type(n) == T_LONG);
 7441   match(Set dst (RoundVD src));
 7442   effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2,  KILL cr);
 7443   format %{ "vector_round_long $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
 7444   ins_encode %{
 7445     int vlen_enc = vector_length_encoding(this);
 7446     InternalAddress new_mxcsr = $constantaddress((jint)0x3F80);
 7447     __ vector_round_double_evex($dst$$XMMRegister, $src$$XMMRegister,
 7448                                 ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), new_mxcsr, vlen_enc,
 7449                                 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
 7450   %}
 7451   ins_pipe( pipe_slow );
 7452 %}
 7453 
 7454 #endif // _LP64
 7455 
 7456 // --------------------------------- VectorMaskCmp --------------------------------------
 7457 
 7458 instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
 7459   predicate(n->bottom_type()->isa_vectmask() == NULL &&
 7460             Matcher::vector_length_in_bytes(n->in(1)->in(1)) >=  8 && // src1
 7461             Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
 7462             is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
 7463   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7464   format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
 7465   ins_encode %{
 7466     int vlen_enc = vector_length_encoding(this, $src1);
 7467     Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
 7468     if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
 7469       __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7470     } else {
 7471       __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7472     }
 7473   %}
 7474   ins_pipe( pipe_slow );
 7475 %}
 7476 
 7477 instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
 7478   predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
 7479             n->bottom_type()->isa_vectmask() == NULL &&
 7480             is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
 7481   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7482   effect(TEMP ktmp);
 7483   format %{ "vector_compare $dst,$src1,$src2,$cond" %}
 7484   ins_encode %{
 7485     int vlen_enc = Assembler::AVX_512bit;
 7486     Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
 7487     KRegister mask = k0; // The comparison itself is not being masked.
 7488     if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
 7489       __ evcmpps($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7490       __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
 7491     } else {
 7492       __ evcmppd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7493       __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
 7494     }
 7495   %}
 7496   ins_pipe( pipe_slow );
 7497 %}
 7498 
 7499 instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
 7500   predicate(n->bottom_type()->isa_vectmask() &&
 7501             is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
 7502   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7503   format %{ "vector_compare_evex $dst,$src1,$src2,$cond\t!" %}
 7504   ins_encode %{
 7505     assert(bottom_type()->isa_vectmask(), "TypeVectMask expected");
 7506     int vlen_enc = vector_length_encoding(this, $src1);
 7507     Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
 7508     KRegister mask = k0; // The comparison itself is not being masked.
 7509     if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
 7510       __ evcmpps($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7511     } else {
 7512       __ evcmppd($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7513     }
 7514   %}
 7515   ins_pipe( pipe_slow );
 7516 %}
 7517 
 7518 instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
 7519   predicate(n->bottom_type()->isa_vectmask() == NULL &&
 7520             !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
 7521             Matcher::vector_length_in_bytes(n->in(1)->in(1)) >=  4 && // src1
 7522             Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
 7523             is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
 7524             (n->in(2)->get_int() == BoolTest::eq ||
 7525              n->in(2)->get_int() == BoolTest::lt ||
 7526              n->in(2)->get_int() == BoolTest::gt)); // cond
 7527   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7528   format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
 7529   ins_encode %{
 7530     int vlen_enc = vector_length_encoding(this, $src1);
 7531     Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
 7532     Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
 7533     __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, cmp, ww, vlen_enc);
 7534   %}
 7535   ins_pipe( pipe_slow );
 7536 %}
 7537 
 7538 instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
 7539   predicate(n->bottom_type()->isa_vectmask() == NULL &&
 7540             !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
 7541             Matcher::vector_length_in_bytes(n->in(1)->in(1)) >=  4 && // src1
 7542             Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
 7543             is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
 7544             (n->in(2)->get_int() == BoolTest::ne ||
 7545              n->in(2)->get_int() == BoolTest::le ||
 7546              n->in(2)->get_int() == BoolTest::ge)); // cond
 7547   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7548   effect(TEMP dst, TEMP xtmp);
 7549   format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
 7550   ins_encode %{
 7551     int vlen_enc = vector_length_encoding(this, $src1);
 7552     Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
 7553     Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
 7554     __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
 7555   %}
 7556   ins_pipe( pipe_slow );
 7557 %}
 7558 
 7559 instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
 7560   predicate(n->bottom_type()->isa_vectmask() == NULL &&
 7561             Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
 7562             Matcher::vector_length_in_bytes(n->in(1)->in(1)) >=  4 && // src1
 7563             Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
 7564             is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
 7565   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7566   effect(TEMP dst, TEMP xtmp);
 7567   format %{ "vector_compareu $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
 7568   ins_encode %{
 7569     InternalAddress flip_bit = $constantaddress(high_bit_set(Matcher::vector_element_basic_type(this, $src1)));
 7570     int vlen_enc = vector_length_encoding(this, $src1);
 7571     Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
 7572     Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
 7573 
 7574     if (vlen_enc == Assembler::AVX_128bit) {
 7575       __ vmovddup($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
 7576     } else {
 7577       __ vbroadcastsd($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
 7578     }
 7579     __ vpxor($dst$$XMMRegister, $xtmp$$XMMRegister, $src1$$XMMRegister, vlen_enc);
 7580     __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $src2$$XMMRegister, vlen_enc);
 7581     __ vpcmpCCW($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
 7582   %}
 7583   ins_pipe( pipe_slow );
 7584 %}
 7585 
 7586 instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
 7587   predicate((n->bottom_type()->isa_vectmask() == NULL &&
 7588              Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
 7589              is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
 7590   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7591   effect(TEMP ktmp);
 7592   format %{ "vector_compare $dst,$src1,$src2,$cond" %}
 7593   ins_encode %{
 7594     assert(UseAVX > 2, "required");
 7595 
 7596     int vlen_enc = vector_length_encoding(this, $src1);
 7597     Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
 7598     bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
 7599     KRegister mask = k0; // The comparison itself is not being masked.
 7600     bool merge = false;
 7601     BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
 7602 
 7603     switch (src1_elem_bt) {
 7604       case T_INT: {
 7605         __ evpcmpd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
 7606         __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
 7607         break;

 7783 // --------------------------------- Vector Blend --------------------------------------
 7784 
 7785 instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
 7786   predicate(UseAVX == 0);
 7787   match(Set dst (VectorBlend (Binary dst src) mask));
 7788   format %{ "vector_blend  $dst,$src,$mask\t! using $tmp as TEMP" %}
 7789   effect(TEMP tmp);
 7790   ins_encode %{
 7791     assert(UseSSE >= 4, "required");
 7792 
 7793     if ($mask$$XMMRegister != $tmp$$XMMRegister) {
 7794       __ movdqu($tmp$$XMMRegister, $mask$$XMMRegister);
 7795     }
 7796     __ pblendvb($dst$$XMMRegister, $src$$XMMRegister); // uses xmm0 as mask
 7797   %}
 7798   ins_pipe( pipe_slow );
 7799 %}
 7800 
 7801 instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
 7802   predicate(UseAVX > 0 &&
 7803             n->in(2)->bottom_type()->isa_vectmask() == NULL &&
 7804             Matcher::vector_length_in_bytes(n) <= 32 &&
 7805             is_integral_type(Matcher::vector_element_basic_type(n)));
 7806   match(Set dst (VectorBlend (Binary src1 src2) mask));
 7807   format %{ "vector_blend  $dst,$src1,$src2,$mask\t!" %}
 7808   ins_encode %{
 7809     int vlen_enc = vector_length_encoding(this);
 7810     __ vpblendvb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
 7811   %}
 7812   ins_pipe( pipe_slow );
 7813 %}
 7814 
 7815 instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
 7816   predicate(UseAVX > 0 &&
 7817             n->in(2)->bottom_type()->isa_vectmask() == NULL &&
 7818             Matcher::vector_length_in_bytes(n) <= 32 &&
 7819             !is_integral_type(Matcher::vector_element_basic_type(n)));
 7820   match(Set dst (VectorBlend (Binary src1 src2) mask));
 7821   format %{ "vector_blend  $dst,$src1,$src2,$mask\t!" %}
 7822   ins_encode %{
 7823     int vlen_enc = vector_length_encoding(this);
 7824     __ vblendvps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
 7825   %}
 7826   ins_pipe( pipe_slow );
 7827 %}
 7828 
 7829 instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
 7830   predicate(Matcher::vector_length_in_bytes(n) == 64 &&
 7831             n->in(2)->bottom_type()->isa_vectmask() == NULL);
 7832   match(Set dst (VectorBlend (Binary src1 src2) mask));
 7833   format %{ "vector_blend  $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
 7834   effect(TEMP ktmp);
 7835   ins_encode %{
 7836      int vlen_enc = Assembler::AVX_512bit;
 7837      BasicType elem_bt = Matcher::vector_element_basic_type(this);
 7838     __ evpcmp(elem_bt, $ktmp$$KRegister, k0, $mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), Assembler::eq, vlen_enc, noreg);
 7839     __ evpblend(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
 7840   %}
 7841   ins_pipe( pipe_slow );
 7842 %}
 7843 
 7844 
 7845 instruct evblendvp64_masked(vec dst, vec src1, vec src2, kReg mask) %{
 7846   predicate(n->in(2)->bottom_type()->isa_vectmask() &&
 7847             (!is_subword_type(Matcher::vector_element_basic_type(n)) ||
 7848              VM_Version::supports_avx512bw()));
 7849   match(Set dst (VectorBlend (Binary src1 src2) mask));
 7850   format %{ "vector_blend  $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
 7851   ins_encode %{

 8028   %}
 8029   ins_pipe( pipe_slow );
 8030 %}
 8031 
 8032 instruct ktest_ge8(rFlagsRegU cr, kReg src1, kReg src2) %{
 8033   predicate(Matcher::vector_length(n->in(1)) >= 16 ||
 8034             (Matcher::vector_length(n->in(1)) == 8 && VM_Version::supports_avx512dq()));
 8035   match(Set cr (VectorTest src1 src2));
 8036   format %{ "ktest_ge8  $src1, $src2\n\t" %}
 8037   ins_encode %{
 8038     uint masklen = Matcher::vector_length(this, $src1);
 8039     __ kortest(masklen, $src1$$KRegister, $src1$$KRegister);
 8040   %}
 8041   ins_pipe( pipe_slow );
 8042 %}
 8043 #endif
 8044 
 8045 //------------------------------------- LoadMask --------------------------------------------
 8046 
 8047 instruct loadMask(legVec dst, legVec src) %{
 8048   predicate(n->bottom_type()->isa_vectmask() == NULL && !VM_Version::supports_avx512vlbw());
 8049   match(Set dst (VectorLoadMask src));
 8050   effect(TEMP dst);
 8051   format %{ "vector_loadmask_byte $dst, $src\n\t" %}
 8052   ins_encode %{
 8053     int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
 8054     BasicType elem_bt = Matcher::vector_element_basic_type(this);
 8055     __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt, true);
 8056   %}
 8057   ins_pipe( pipe_slow );
 8058 %}
 8059 
 8060 instruct loadMask64(kReg dst, vec src, vec xtmp) %{
 8061   predicate(n->bottom_type()->isa_vectmask() && !VM_Version::supports_avx512vlbw());
 8062   match(Set dst (VectorLoadMask src));
 8063   effect(TEMP xtmp);
 8064   format %{ "vector_loadmask_64byte $dst, $src\t! using $xtmp as TEMP" %}
 8065   ins_encode %{
 8066     __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
 8067                         true, Assembler::AVX_512bit);
 8068   %}
 8069   ins_pipe( pipe_slow );
 8070 %}
 8071 
 8072 instruct loadMask_evex(kReg dst, vec src,  vec xtmp) %{
 8073   predicate(n->bottom_type()->isa_vectmask() && VM_Version::supports_avx512vlbw());
 8074   match(Set dst (VectorLoadMask src));
 8075   effect(TEMP xtmp);
 8076   format %{ "vector_loadmask_byte $dst, $src\t! using $xtmp as TEMP" %}
 8077   ins_encode %{
 8078     int vlen_enc = vector_length_encoding(in(1));
 8079     __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
 8080                         false, vlen_enc);
 8081   %}
 8082   ins_pipe( pipe_slow );
 8083 %}
 8084 
 8085 //------------------------------------- StoreMask --------------------------------------------
 8086 
 8087 instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
 8088   predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
 8089   match(Set dst (VectorStoreMask src size));
 8090   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
 8091   ins_encode %{
 8092     int vlen = Matcher::vector_length(this);
 8093     if (vlen <= 16 && UseAVX <= 2) {
 8094       assert(UseSSE >= 3, "required");
 8095       __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
 8096     } else {
 8097       assert(UseAVX > 0, "required");
 8098       int src_vlen_enc = vector_length_encoding(this, $src);
 8099       __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
 8100     }
 8101   %}
 8102   ins_pipe( pipe_slow );
 8103 %}
 8104 
 8105 instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
 8106   predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
 8107   match(Set dst (VectorStoreMask src size));
 8108   effect(TEMP_DEF dst, TEMP xtmp);
 8109   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
 8110   ins_encode %{
 8111     int vlen_enc = Assembler::AVX_128bit;
 8112     int vlen = Matcher::vector_length(this);
 8113     if (vlen <= 8) {
 8114       assert(UseSSE >= 3, "required");
 8115       __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
 8116       __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
 8117       __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
 8118     } else {
 8119       assert(UseAVX > 0, "required");
 8120       __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
 8121       __ vpacksswb($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
 8122       __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
 8123     }
 8124   %}
 8125   ins_pipe( pipe_slow );
 8126 %}
 8127 
 8128 instruct vstoreMask4B(vec dst, vec src, vec xtmp, immI_4 size) %{
 8129   predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
 8130   match(Set dst (VectorStoreMask src size));
 8131   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
 8132   effect(TEMP_DEF dst, TEMP xtmp);
 8133   ins_encode %{
 8134     int vlen_enc = Assembler::AVX_128bit;
 8135     int vlen = Matcher::vector_length(this);
 8136     if (vlen <= 4) {
 8137       assert(UseSSE >= 3, "required");
 8138       __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
 8139       __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
 8140       __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
 8141       __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
 8142     } else {
 8143       assert(UseAVX > 0, "required");
 8144       __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
 8145       __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
 8146       __ vpackssdw($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
 8147       __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
 8148       __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
 8149     }

 8169 
 8170 instruct storeMask8B_avx(vec dst, vec src, immI_8 size, vec vtmp) %{
 8171   predicate(UseAVX <= 2 && Matcher::vector_length(n) == 4);
 8172   match(Set dst (VectorStoreMask src size));
 8173   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s], using $vtmp as TEMP" %}
 8174   effect(TEMP_DEF dst, TEMP vtmp);
 8175   ins_encode %{
 8176     int vlen_enc = Assembler::AVX_128bit;
 8177     __ vshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 0x88, Assembler::AVX_256bit);
 8178     __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
 8179     __ vblendps($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0xC, vlen_enc);
 8180     __ vpxor($vtmp$$XMMRegister, $vtmp$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
 8181     __ vpackssdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
 8182     __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
 8183     __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
 8184   %}
 8185   ins_pipe( pipe_slow );
 8186 %}
 8187 
 8188 instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
 8189   predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
 8190   match(Set dst (VectorStoreMask src size));
 8191   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
 8192   ins_encode %{
 8193     int src_vlen_enc = vector_length_encoding(this, $src);
 8194     int dst_vlen_enc = vector_length_encoding(this);
 8195     if (!VM_Version::supports_avx512vl()) {
 8196       src_vlen_enc = Assembler::AVX_512bit;
 8197     }
 8198     __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
 8199     __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
 8200   %}
 8201   ins_pipe( pipe_slow );
 8202 %}
 8203 
 8204 instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
 8205   predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
 8206   match(Set dst (VectorStoreMask src size));
 8207   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
 8208   ins_encode %{
 8209     int src_vlen_enc = vector_length_encoding(this, $src);
 8210     int dst_vlen_enc = vector_length_encoding(this);
 8211     if (!VM_Version::supports_avx512vl()) {
 8212       src_vlen_enc = Assembler::AVX_512bit;
 8213     }
 8214     __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
 8215     __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
 8216   %}
 8217   ins_pipe( pipe_slow );
 8218 %}
 8219 
 8220 instruct vstoreMask_evex_vectmask(vec dst, kReg mask, immI size) %{
 8221   predicate(n->in(1)->bottom_type()->isa_vectmask() && !VM_Version::supports_avx512vlbw());
 8222   match(Set dst (VectorStoreMask mask size));
 8223   effect(TEMP_DEF dst);
 8224   format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
 8225   ins_encode %{

 9007 %}
 9008 
 9009 instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
 9010   predicate(n->in(1)->bottom_type()->isa_vectmask());
 9011   match(Set dst (VectorMaskToLong mask));
 9012   effect(TEMP dst, KILL cr);
 9013   format %{ "vector_tolong_evex $dst, $mask \t! vector mask tolong" %}
 9014   ins_encode %{
 9015     int opcode = this->ideal_Opcode();
 9016     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9017     int mask_len = Matcher::vector_length(this, $mask);
 9018     int mask_size = mask_len * type2aelembytes(mbt);
 9019     int vlen_enc = vector_length_encoding(this, $mask);
 9020     __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
 9021                              $dst$$Register, mask_len, mask_size, vlen_enc);
 9022   %}
 9023   ins_pipe( pipe_slow );
 9024 %}
 9025 
 9026 instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
 9027   predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL);
 9028   match(Set dst (VectorMaskToLong mask));
 9029   format %{ "vector_tolong_bool $dst, $mask \t! using $xtmp as TEMP" %}
 9030   effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
 9031   ins_encode %{
 9032     int opcode = this->ideal_Opcode();
 9033     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9034     int mask_len = Matcher::vector_length(this, $mask);
 9035     int vlen_enc = vector_length_encoding(this, $mask);
 9036     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9037                              $dst$$Register, mask_len, mbt, vlen_enc);
 9038   %}
 9039   ins_pipe( pipe_slow );
 9040 %}
 9041 
 9042 instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr) %{
 9043   predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == NULL);
 9044   match(Set dst (VectorMaskToLong (VectorStoreMask mask size)));
 9045   format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
 9046   effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
 9047   ins_encode %{
 9048     int opcode = this->ideal_Opcode();
 9049     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9050     int mask_len = Matcher::vector_length(this, $mask);
 9051     int vlen_enc = vector_length_encoding(this, $mask);
 9052     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9053                              $dst$$Register, mask_len, mbt, vlen_enc);
 9054   %}
 9055   ins_pipe( pipe_slow );
 9056 %}
 9057 
 9058 instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
 9059   predicate(n->in(1)->bottom_type()->isa_vectmask());
 9060   match(Set dst (VectorMaskTrueCount mask));
 9061   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
 9062   format %{ "vector_truecount_evex $dst, $mask \t! using $tmp as TEMP" %}
 9063   ins_encode %{
 9064     int opcode = this->ideal_Opcode();
 9065     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9066     int mask_len = Matcher::vector_length(this, $mask);
 9067     int mask_size = mask_len * type2aelembytes(mbt);
 9068     int vlen_enc = vector_length_encoding(this, $mask);
 9069     __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
 9070                              $tmp$$Register, mask_len, mask_size, vlen_enc);
 9071   %}
 9072   ins_pipe( pipe_slow );
 9073 %}
 9074 
 9075 instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
 9076   predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL);
 9077   match(Set dst (VectorMaskTrueCount mask));
 9078   effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
 9079   format %{ "vector_truecount_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
 9080   ins_encode %{
 9081     int opcode = this->ideal_Opcode();
 9082     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9083     int mask_len = Matcher::vector_length(this, $mask);
 9084     int vlen_enc = vector_length_encoding(this, $mask);
 9085     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9086                              $tmp$$Register, mask_len, mbt, vlen_enc);
 9087   %}
 9088   ins_pipe( pipe_slow );
 9089 %}
 9090 
 9091 instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
 9092   predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == NULL);
 9093   match(Set dst (VectorMaskTrueCount (VectorStoreMask mask size)));
 9094   effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
 9095   format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
 9096   ins_encode %{
 9097     int opcode = this->ideal_Opcode();
 9098     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9099     int mask_len = Matcher::vector_length(this, $mask);
 9100     int vlen_enc = vector_length_encoding(this, $mask);
 9101     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9102                              $tmp$$Register, mask_len, mbt, vlen_enc);
 9103   %}
 9104   ins_pipe( pipe_slow );
 9105 %}
 9106 
 9107 instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
 9108   predicate(n->in(1)->bottom_type()->isa_vectmask());
 9109   match(Set dst (VectorMaskFirstTrue mask));
 9110   match(Set dst (VectorMaskLastTrue mask));
 9111   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
 9112   format %{ "vector_mask_first_or_last_true_evex $dst, $mask \t! using $tmp as TEMP" %}
 9113   ins_encode %{
 9114     int opcode = this->ideal_Opcode();
 9115     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9116     int mask_len = Matcher::vector_length(this, $mask);
 9117     int mask_size = mask_len * type2aelembytes(mbt);
 9118     int vlen_enc = vector_length_encoding(this, $mask);
 9119     __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
 9120                              $tmp$$Register, mask_len, mask_size, vlen_enc);
 9121   %}
 9122   ins_pipe( pipe_slow );
 9123 %}
 9124 
 9125 instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
 9126   predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL);
 9127   match(Set dst (VectorMaskFirstTrue mask));
 9128   match(Set dst (VectorMaskLastTrue mask));
 9129   effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
 9130   format %{ "vector_mask_first_or_last_true_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
 9131   ins_encode %{
 9132     int opcode = this->ideal_Opcode();
 9133     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9134     int mask_len = Matcher::vector_length(this, $mask);
 9135     int vlen_enc = vector_length_encoding(this, $mask);
 9136     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9137                              $tmp$$Register, mask_len, mbt, vlen_enc);
 9138   %}
 9139   ins_pipe( pipe_slow );
 9140 %}
 9141 
 9142 instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
 9143   predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == NULL);
 9144   match(Set dst (VectorMaskFirstTrue (VectorStoreMask mask size)));
 9145   match(Set dst (VectorMaskLastTrue (VectorStoreMask mask size)));
 9146   effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
 9147   format %{ "vector_mask_first_or_last_true_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
 9148   ins_encode %{
 9149     int opcode = this->ideal_Opcode();
 9150     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9151     int mask_len = Matcher::vector_length(this, $mask);
 9152     int vlen_enc = vector_length_encoding(this, $mask);
 9153     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9154                              $tmp$$Register, mask_len, mbt, vlen_enc);
 9155   %}
 9156   ins_pipe( pipe_slow );
 9157 %}
 9158 
 9159 // --------------------------------- Compress/Expand Operations ---------------------------
 9160 
 9161 instruct vcompress_expand_reg_evex(vec dst, vec src, kReg mask) %{
 9162   match(Set dst (CompressV src mask));
 9163   match(Set dst (ExpandV src mask));

 9958     uint masklen = Matcher::vector_length(this);
 9959     __ knot(masklen, $dst$$KRegister, $src$$KRegister, $ktmp$$KRegister, $rtmp$$Register);
 9960   %}
 9961   ins_pipe( pipe_slow );
 9962 %}
 9963 
 9964 instruct mask_not_imm(kReg dst, kReg src, immI_M1 cnt) %{
 9965   predicate((Matcher::vector_length(n) == 8 && VM_Version::supports_avx512dq()) ||
 9966             (Matcher::vector_length(n) == 16) ||
 9967             (Matcher::vector_length(n) > 16 && VM_Version::supports_avx512bw()));
 9968   match(Set dst (XorVMask src (MaskAll cnt)));
 9969   format %{ "mask_not $dst, $src, $cnt \t! mask not operation" %}
 9970   ins_encode %{
 9971     uint masklen = Matcher::vector_length(this);
 9972     __ knot(masklen, $dst$$KRegister, $src$$KRegister);
 9973   %}
 9974   ins_pipe( pipe_slow );
 9975 %}
 9976 
 9977 instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp) %{
 9978   predicate(n->bottom_type()->isa_vectmask() == NULL && Matcher::vector_length(n) <= 8);
 9979   match(Set dst (VectorLongToMask src));
 9980   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp);
 9981   format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp as TEMP" %}
 9982   ins_encode %{
 9983     int mask_len = Matcher::vector_length(this);
 9984     int vec_enc  = vector_length_encoding(mask_len);
 9985     __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
 9986                               $rtmp2$$Register, xnoreg, mask_len, vec_enc);
 9987   %}
 9988   ins_pipe( pipe_slow );
 9989 %}
 9990 
 9991 
 9992 instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp1, rFlagsReg cr) %{
 9993   predicate(n->bottom_type()->isa_vectmask() == NULL && Matcher::vector_length(n) > 8);
 9994   match(Set dst (VectorLongToMask src));
 9995   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp1, KILL cr);
 9996   format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp1, as TEMP" %}
 9997   ins_encode %{
 9998     int mask_len = Matcher::vector_length(this);
 9999     assert(mask_len <= 32, "invalid mask length");
10000     int vec_enc  = vector_length_encoding(mask_len);
10001     __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
10002                               $rtmp2$$Register, $xtmp1$$XMMRegister, mask_len, vec_enc);
10003   %}
10004   ins_pipe( pipe_slow );
10005 %}
10006 
10007 instruct long_to_mask_evex(kReg dst, rRegL src) %{
10008   predicate(n->bottom_type()->isa_vectmask());
10009   match(Set dst (VectorLongToMask src));
10010   format %{ "long_to_mask_evex $dst, $src\t!" %}
10011   ins_encode %{
10012     __ kmov($dst$$KRegister, $src$$Register);
10013   %}

 1285 int MachNode::compute_padding(int current_offset) const {
 1286   if (flags() & Node::PD::Flag_intel_jcc_erratum) {
 1287     Compile* C = Compile::current();
 1288     PhaseOutput* output = C->output();
 1289     Block* block = output->block();
 1290     int index = output->index();
 1291     return IntelJccErratum::compute_padding(current_offset, this, block, index, C->regalloc());
 1292   } else {
 1293     return 0;
 1294   }
 1295 }
 1296 
 1297 // Emit exception handler code.
 1298 // Stuff framesize into a register and call a VM stub routine.
 1299 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
 1300 
 1301   // Note that the code buffer's insts_mark is always relative to insts.
 1302   // That's why we must use the macroassembler to generate a handler.
 1303   C2_MacroAssembler _masm(&cbuf);
 1304   address base = __ start_a_stub(size_exception_handler());
 1305   if (base == nullptr) {
 1306     ciEnv::current()->record_failure("CodeCache is full");
 1307     return 0;  // CodeBuffer::expand failed
 1308   }
 1309   int offset = __ offset();
 1310   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 1311   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 1312   __ end_a_stub();
 1313   return offset;
 1314 }
 1315 
 1316 // Emit deopt handler code.
 1317 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
 1318 
 1319   // Note that the code buffer's insts_mark is always relative to insts.
 1320   // That's why we must use the macroassembler to generate a handler.
 1321   C2_MacroAssembler _masm(&cbuf);
 1322   address base = __ start_a_stub(size_deopt_handler());
 1323   if (base == nullptr) {
 1324     ciEnv::current()->record_failure("CodeCache is full");
 1325     return 0;  // CodeBuffer::expand failed
 1326   }
 1327   int offset = __ offset();
 1328 
 1329 #ifdef _LP64
 1330   address the_pc = (address) __ pc();
 1331   Label next;
 1332   // push a "the_pc" on the stack without destroying any registers
 1333   // as they all may be live.
 1334 
 1335   // push address of "next"
 1336   __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
 1337   __ bind(next);
 1338   // adjust it so it matches "the_pc"
 1339   __ subptr(Address(rsp, 0), __ offset() - offset);
 1340 #else
 1341   InternalAddress here(__ pc());
 1342   __ pushptr(here.addr(), noreg);
 1343 #endif

 2160     return new legVecZOper();
 2161   }
 2162   if (legacy) {
 2163     switch (ideal_reg) {
 2164       case Op_VecS: return new legVecSOper();
 2165       case Op_VecD: return new legVecDOper();
 2166       case Op_VecX: return new legVecXOper();
 2167       case Op_VecY: return new legVecYOper();
 2168       case Op_VecZ: return new legVecZOper();
 2169     }
 2170   } else {
 2171     switch (ideal_reg) {
 2172       case Op_VecS: return new vecSOper();
 2173       case Op_VecD: return new vecDOper();
 2174       case Op_VecX: return new vecXOper();
 2175       case Op_VecY: return new vecYOper();
 2176       case Op_VecZ: return new vecZOper();
 2177     }
 2178   }
 2179   ShouldNotReachHere();
 2180   return nullptr;
 2181 }
 2182 
 2183 bool Matcher::is_reg2reg_move(MachNode* m) {
 2184   switch (m->rule()) {
 2185     case MoveVec2Leg_rule:
 2186     case MoveLeg2Vec_rule:
 2187     case MoveF2VL_rule:
 2188     case MoveF2LEG_rule:
 2189     case MoveVL2F_rule:
 2190     case MoveLEG2F_rule:
 2191     case MoveD2VL_rule:
 2192     case MoveD2LEG_rule:
 2193     case MoveVL2D_rule:
 2194     case MoveLEG2D_rule:
 2195       return true;
 2196     default:
 2197       return false;
 2198   }
 2199 }
 2200 

 2329   }
 2330   return false;
 2331 }
 2332 
 2333 // This function identifies sub-graphs in which a 'load' node is
 2334 // input to two different nodes, and such that it can be matched
 2335 // with BMI instructions like blsi, blsr, etc.
 2336 // Example : for b = -a[i] & a[i] can be matched to blsi r32, m32.
 2337 // The graph is (AndL (SubL Con0 LoadL*) LoadL*), where LoadL*
 2338 // refers to the same node.
 2339 //
 2340 // Match the generic fused operations pattern (op1 (op2 Con{ConType} mop) mop)
 2341 // This is a temporary solution until we make DAGs expressible in ADL.
 2342 template<typename ConType>
 2343 class FusedPatternMatcher {
 2344   Node* _op1_node;
 2345   Node* _mop_node;
 2346   int _con_op;
 2347 
 2348   static int match_next(Node* n, int next_op, int next_op_idx) {
 2349     if (n->in(1) == nullptr || n->in(2) == nullptr) {
 2350       return -1;
 2351     }
 2352 
 2353     if (next_op_idx == -1) { // n is commutative, try rotations
 2354       if (n->in(1)->Opcode() == next_op) {
 2355         return 1;
 2356       } else if (n->in(2)->Opcode() == next_op) {
 2357         return 2;
 2358       }
 2359     } else {
 2360       assert(next_op_idx > 0 && next_op_idx <= 2, "Bad argument index");
 2361       if (n->in(next_op_idx)->Opcode() == next_op) {
 2362         return next_op_idx;
 2363       }
 2364     }
 2365     return -1;
 2366   }
 2367 
 2368  public:
 2369   FusedPatternMatcher(Node* op1_node, Node* mop_node, int con_op) :

 2396       if (op2_con_idx == -1) {
 2397         return false;
 2398       }
 2399       // Memory operation must be the other edge
 2400       int op2_mop_idx = (op2_con_idx & 1) + 1;
 2401       // Check that the memory operation is the same node
 2402       if (op2_node->in(op2_mop_idx) == _mop_node) {
 2403         // Now check the constant
 2404         const Type* con_type = op2_node->in(op2_con_idx)->bottom_type();
 2405         if (con_type != Type::TOP && ConType::as_self(con_type)->get_con() == con_value) {
 2406           return true;
 2407         }
 2408       }
 2409     }
 2410     return false;
 2411   }
 2412 };
 2413 
 2414 static bool is_bmi_pattern(Node* n, Node* m) {
 2415   assert(UseBMI1Instructions, "sanity");
 2416   if (n != nullptr && m != nullptr) {
 2417     if (m->Opcode() == Op_LoadI) {
 2418       FusedPatternMatcher<TypeInt> bmii(n, m, Op_ConI);
 2419       return bmii.match(Op_AndI, -1, Op_SubI,  1,  0)  ||
 2420              bmii.match(Op_AndI, -1, Op_AddI, -1, -1)  ||
 2421              bmii.match(Op_XorI, -1, Op_AddI, -1, -1);
 2422     } else if (m->Opcode() == Op_LoadL) {
 2423       FusedPatternMatcher<TypeLong> bmil(n, m, Op_ConL);
 2424       return bmil.match(Op_AndL, -1, Op_SubL,  1,  0) ||
 2425              bmil.match(Op_AndL, -1, Op_AddL, -1, -1) ||
 2426              bmil.match(Op_XorL, -1, Op_AddL, -1, -1);
 2427     }
 2428   }
 2429   return false;
 2430 }
 2431 
 2432 // Should the matcher clone input 'm' of node 'n'?
 2433 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2434   // If 'n' and 'm' are part of a graph for BMI instruction, clone the input 'm'.
 2435   if (UseBMI1Instructions && is_bmi_pattern(n, m)) {
 2436     mstack.push(m, Visit);

 2762   uint MachBreakpointNode::size(PhaseRegAlloc* ra_) const {
 2763     return MachNode::size(ra_);
 2764   }
 2765 
 2766 %}
 2767 
 2768 encode %{
 2769 
 2770   enc_class call_epilog %{
 2771     C2_MacroAssembler _masm(&cbuf);
 2772     if (VerifyStackAtCalls) {
 2773       // Check that stack depth is unchanged: find majik cookie on stack
 2774       int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
 2775       Label L;
 2776       __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
 2777       __ jccb(Assembler::equal, L);
 2778       // Die if stack mismatch
 2779       __ int3();
 2780       __ bind(L);
 2781     }
 2782     if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
 2783       C2_MacroAssembler _masm(&cbuf);
 2784       if (!_method->signature()->returns_null_free_inline_type()) {
 2785         // The last return value is not set by the callee but used to pass IsInit information to compiled code.
 2786         // Search for the corresponding projection, get the register and emit code that initialized it.
 2787         uint con = (tf()->range_cc()->cnt() - 1);
 2788         for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
 2789           ProjNode* proj = fast_out(i)->as_Proj();
 2790           if (proj->_con == con) {
 2791             // Set IsInit if rax is non-null (a non-null value is returned buffered or scalarized)
 2792             OptoReg::Name optoReg = ra_->get_reg_first(proj);
 2793             VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
 2794             Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
 2795             __ testq(rax, rax);
 2796             __ setb(Assembler::notZero, toReg);
 2797             __ movzbl(toReg, toReg);
 2798             if (reg->is_stack()) {
 2799               int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
 2800               __ movq(Address(rsp, st_off), toReg);
 2801             }
 2802             break;
 2803           }
 2804         }
 2805       }
 2806       if (return_value_is_used()) {
 2807         // An inline type is returned as fields in multiple registers.
 2808         // Rax either contains an oop if the inline type is buffered or a pointer
 2809         // to the corresponding InlineKlass with the lowest bit set to 1. Zero rax
 2810         // if the lowest bit is set to allow C2 to use the oop after null checking.
 2811         // rax &= (rax & 1) - 1
 2812         __ movptr(rscratch1, rax);
 2813         __ andptr(rscratch1, 0x1);
 2814         __ subptr(rscratch1, 0x1);
 2815         __ andptr(rax, rscratch1);
 2816       }
 2817     }
 2818   %}
 2819 
 2820 %}
 2821 
 2822 // Operands for bound floating pointer register arguments
 2823 operand rxmm0() %{
 2824   constraint(ALLOC_IN_RC(xmm0_reg));
 2825   match(VecX);
 2826   format%{%}
 2827   interface(REG_INTER);
 2828 %}
 2829 
 2830 //----------OPERANDS-----------------------------------------------------------
 2831 // Operand definitions must precede instruction definitions for correct parsing
 2832 // in the ADLC because operands constitute user defined types which are used in
 2833 // instruction definitions.
 2834 
 2835 // Vectors
 2836 
 2837 // Dummy generic vector class. Should be used for all vector operands.

 7475 instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
 7476   predicate(Matcher::vector_element_basic_type(n) == T_LONG);
 7477   match(Set dst (RoundVD src));
 7478   effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2,  KILL cr);
 7479   format %{ "vector_round_long $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
 7480   ins_encode %{
 7481     int vlen_enc = vector_length_encoding(this);
 7482     InternalAddress new_mxcsr = $constantaddress((jint)0x3F80);
 7483     __ vector_round_double_evex($dst$$XMMRegister, $src$$XMMRegister,
 7484                                 ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), new_mxcsr, vlen_enc,
 7485                                 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
 7486   %}
 7487   ins_pipe( pipe_slow );
 7488 %}
 7489 
 7490 #endif // _LP64
 7491 
 7492 // --------------------------------- VectorMaskCmp --------------------------------------
 7493 
 7494 instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
 7495   predicate(n->bottom_type()->isa_vectmask() == nullptr &&
 7496             Matcher::vector_length_in_bytes(n->in(1)->in(1)) >=  8 && // src1
 7497             Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
 7498             is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
 7499   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7500   format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
 7501   ins_encode %{
 7502     int vlen_enc = vector_length_encoding(this, $src1);
 7503     Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
 7504     if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
 7505       __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7506     } else {
 7507       __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7508     }
 7509   %}
 7510   ins_pipe( pipe_slow );
 7511 %}
 7512 
 7513 instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
 7514   predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
 7515             n->bottom_type()->isa_vectmask() == nullptr &&
 7516             is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
 7517   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7518   effect(TEMP ktmp);
 7519   format %{ "vector_compare $dst,$src1,$src2,$cond" %}
 7520   ins_encode %{
 7521     int vlen_enc = Assembler::AVX_512bit;
 7522     Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
 7523     KRegister mask = k0; // The comparison itself is not being masked.
 7524     if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
 7525       __ evcmpps($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7526       __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
 7527     } else {
 7528       __ evcmppd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7529       __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
 7530     }
 7531   %}
 7532   ins_pipe( pipe_slow );
 7533 %}
 7534 
 7535 instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
 7536   predicate(n->bottom_type()->isa_vectmask() &&
 7537             is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
 7538   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7539   format %{ "vector_compare_evex $dst,$src1,$src2,$cond\t!" %}
 7540   ins_encode %{
 7541     assert(bottom_type()->isa_vectmask(), "TypeVectMask expected");
 7542     int vlen_enc = vector_length_encoding(this, $src1);
 7543     Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
 7544     KRegister mask = k0; // The comparison itself is not being masked.
 7545     if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
 7546       __ evcmpps($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7547     } else {
 7548       __ evcmppd($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
 7549     }
 7550   %}
 7551   ins_pipe( pipe_slow );
 7552 %}
 7553 
 7554 instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
 7555   predicate(n->bottom_type()->isa_vectmask() == nullptr &&
 7556             !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
 7557             Matcher::vector_length_in_bytes(n->in(1)->in(1)) >=  4 && // src1
 7558             Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
 7559             is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
 7560             (n->in(2)->get_int() == BoolTest::eq ||
 7561              n->in(2)->get_int() == BoolTest::lt ||
 7562              n->in(2)->get_int() == BoolTest::gt)); // cond
 7563   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7564   format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
 7565   ins_encode %{
 7566     int vlen_enc = vector_length_encoding(this, $src1);
 7567     Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
 7568     Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
 7569     __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, cmp, ww, vlen_enc);
 7570   %}
 7571   ins_pipe( pipe_slow );
 7572 %}
 7573 
 7574 instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
 7575   predicate(n->bottom_type()->isa_vectmask() == nullptr &&
 7576             !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
 7577             Matcher::vector_length_in_bytes(n->in(1)->in(1)) >=  4 && // src1
 7578             Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
 7579             is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
 7580             (n->in(2)->get_int() == BoolTest::ne ||
 7581              n->in(2)->get_int() == BoolTest::le ||
 7582              n->in(2)->get_int() == BoolTest::ge)); // cond
 7583   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7584   effect(TEMP dst, TEMP xtmp);
 7585   format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
 7586   ins_encode %{
 7587     int vlen_enc = vector_length_encoding(this, $src1);
 7588     Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
 7589     Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
 7590     __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
 7591   %}
 7592   ins_pipe( pipe_slow );
 7593 %}
 7594 
 7595 instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
 7596   predicate(n->bottom_type()->isa_vectmask() == nullptr &&
 7597             Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
 7598             Matcher::vector_length_in_bytes(n->in(1)->in(1)) >=  4 && // src1
 7599             Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
 7600             is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
 7601   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7602   effect(TEMP dst, TEMP xtmp);
 7603   format %{ "vector_compareu $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
 7604   ins_encode %{
 7605     InternalAddress flip_bit = $constantaddress(high_bit_set(Matcher::vector_element_basic_type(this, $src1)));
 7606     int vlen_enc = vector_length_encoding(this, $src1);
 7607     Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
 7608     Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
 7609 
 7610     if (vlen_enc == Assembler::AVX_128bit) {
 7611       __ vmovddup($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
 7612     } else {
 7613       __ vbroadcastsd($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
 7614     }
 7615     __ vpxor($dst$$XMMRegister, $xtmp$$XMMRegister, $src1$$XMMRegister, vlen_enc);
 7616     __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $src2$$XMMRegister, vlen_enc);
 7617     __ vpcmpCCW($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
 7618   %}
 7619   ins_pipe( pipe_slow );
 7620 %}
 7621 
 7622 instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
 7623   predicate((n->bottom_type()->isa_vectmask() == nullptr &&
 7624              Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
 7625              is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
 7626   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 7627   effect(TEMP ktmp);
 7628   format %{ "vector_compare $dst,$src1,$src2,$cond" %}
 7629   ins_encode %{
 7630     assert(UseAVX > 2, "required");
 7631 
 7632     int vlen_enc = vector_length_encoding(this, $src1);
 7633     Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
 7634     bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
 7635     KRegister mask = k0; // The comparison itself is not being masked.
 7636     bool merge = false;
 7637     BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
 7638 
 7639     switch (src1_elem_bt) {
 7640       case T_INT: {
 7641         __ evpcmpd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
 7642         __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
 7643         break;

 7819 // --------------------------------- Vector Blend --------------------------------------
 7820 
 7821 instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
 7822   predicate(UseAVX == 0);
 7823   match(Set dst (VectorBlend (Binary dst src) mask));
 7824   format %{ "vector_blend  $dst,$src,$mask\t! using $tmp as TEMP" %}
 7825   effect(TEMP tmp);
 7826   ins_encode %{
 7827     assert(UseSSE >= 4, "required");
 7828 
 7829     if ($mask$$XMMRegister != $tmp$$XMMRegister) {
 7830       __ movdqu($tmp$$XMMRegister, $mask$$XMMRegister);
 7831     }
 7832     __ pblendvb($dst$$XMMRegister, $src$$XMMRegister); // uses xmm0 as mask
 7833   %}
 7834   ins_pipe( pipe_slow );
 7835 %}
 7836 
 7837 instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
 7838   predicate(UseAVX > 0 &&
 7839             n->in(2)->bottom_type()->isa_vectmask() == nullptr &&
 7840             Matcher::vector_length_in_bytes(n) <= 32 &&
 7841             is_integral_type(Matcher::vector_element_basic_type(n)));
 7842   match(Set dst (VectorBlend (Binary src1 src2) mask));
 7843   format %{ "vector_blend  $dst,$src1,$src2,$mask\t!" %}
 7844   ins_encode %{
 7845     int vlen_enc = vector_length_encoding(this);
 7846     __ vpblendvb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
 7847   %}
 7848   ins_pipe( pipe_slow );
 7849 %}
 7850 
 7851 instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
 7852   predicate(UseAVX > 0 &&
 7853             n->in(2)->bottom_type()->isa_vectmask() == nullptr &&
 7854             Matcher::vector_length_in_bytes(n) <= 32 &&
 7855             !is_integral_type(Matcher::vector_element_basic_type(n)));
 7856   match(Set dst (VectorBlend (Binary src1 src2) mask));
 7857   format %{ "vector_blend  $dst,$src1,$src2,$mask\t!" %}
 7858   ins_encode %{
 7859     int vlen_enc = vector_length_encoding(this);
 7860     __ vblendvps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
 7861   %}
 7862   ins_pipe( pipe_slow );
 7863 %}
 7864 
 7865 instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
 7866   predicate(Matcher::vector_length_in_bytes(n) == 64 &&
 7867             n->in(2)->bottom_type()->isa_vectmask() == nullptr);
 7868   match(Set dst (VectorBlend (Binary src1 src2) mask));
 7869   format %{ "vector_blend  $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
 7870   effect(TEMP ktmp);
 7871   ins_encode %{
 7872      int vlen_enc = Assembler::AVX_512bit;
 7873      BasicType elem_bt = Matcher::vector_element_basic_type(this);
 7874     __ evpcmp(elem_bt, $ktmp$$KRegister, k0, $mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), Assembler::eq, vlen_enc, noreg);
 7875     __ evpblend(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
 7876   %}
 7877   ins_pipe( pipe_slow );
 7878 %}
 7879 
 7880 
 7881 instruct evblendvp64_masked(vec dst, vec src1, vec src2, kReg mask) %{
 7882   predicate(n->in(2)->bottom_type()->isa_vectmask() &&
 7883             (!is_subword_type(Matcher::vector_element_basic_type(n)) ||
 7884              VM_Version::supports_avx512bw()));
 7885   match(Set dst (VectorBlend (Binary src1 src2) mask));
 7886   format %{ "vector_blend  $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
 7887   ins_encode %{

 8064   %}
 8065   ins_pipe( pipe_slow );
 8066 %}
 8067 
 8068 instruct ktest_ge8(rFlagsRegU cr, kReg src1, kReg src2) %{
 8069   predicate(Matcher::vector_length(n->in(1)) >= 16 ||
 8070             (Matcher::vector_length(n->in(1)) == 8 && VM_Version::supports_avx512dq()));
 8071   match(Set cr (VectorTest src1 src2));
 8072   format %{ "ktest_ge8  $src1, $src2\n\t" %}
 8073   ins_encode %{
 8074     uint masklen = Matcher::vector_length(this, $src1);
 8075     __ kortest(masklen, $src1$$KRegister, $src1$$KRegister);
 8076   %}
 8077   ins_pipe( pipe_slow );
 8078 %}
 8079 #endif
 8080 
 8081 //------------------------------------- LoadMask --------------------------------------------
 8082 
 8083 instruct loadMask(legVec dst, legVec src) %{
 8084   predicate(n->bottom_type()->isa_vectmask() == nullptr && !VM_Version::supports_avx512vlbw());
 8085   match(Set dst (VectorLoadMask src));
 8086   effect(TEMP dst);
 8087   format %{ "vector_loadmask_byte $dst, $src\n\t" %}
 8088   ins_encode %{
 8089     int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
 8090     BasicType elem_bt = Matcher::vector_element_basic_type(this);
 8091     __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt, true);
 8092   %}
 8093   ins_pipe( pipe_slow );
 8094 %}
 8095 
 8096 instruct loadMask64(kReg dst, vec src, vec xtmp) %{
 8097   predicate(n->bottom_type()->isa_vectmask() && !VM_Version::supports_avx512vlbw());
 8098   match(Set dst (VectorLoadMask src));
 8099   effect(TEMP xtmp);
 8100   format %{ "vector_loadmask_64byte $dst, $src\t! using $xtmp as TEMP" %}
 8101   ins_encode %{
 8102     __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
 8103                         true, Assembler::AVX_512bit);
 8104   %}
 8105   ins_pipe( pipe_slow );
 8106 %}
 8107 
 8108 instruct loadMask_evex(kReg dst, vec src,  vec xtmp) %{
 8109   predicate(n->bottom_type()->isa_vectmask() && VM_Version::supports_avx512vlbw());
 8110   match(Set dst (VectorLoadMask src));
 8111   effect(TEMP xtmp);
 8112   format %{ "vector_loadmask_byte $dst, $src\t! using $xtmp as TEMP" %}
 8113   ins_encode %{
 8114     int vlen_enc = vector_length_encoding(in(1));
 8115     __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
 8116                         false, vlen_enc);
 8117   %}
 8118   ins_pipe( pipe_slow );
 8119 %}
 8120 
 8121 //------------------------------------- StoreMask --------------------------------------------
 8122 
 8123 instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
 8124   predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
 8125   match(Set dst (VectorStoreMask src size));
 8126   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
 8127   ins_encode %{
 8128     int vlen = Matcher::vector_length(this);
 8129     if (vlen <= 16 && UseAVX <= 2) {
 8130       assert(UseSSE >= 3, "required");
 8131       __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
 8132     } else {
 8133       assert(UseAVX > 0, "required");
 8134       int src_vlen_enc = vector_length_encoding(this, $src);
 8135       __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
 8136     }
 8137   %}
 8138   ins_pipe( pipe_slow );
 8139 %}
 8140 
 8141 instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
 8142   predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
 8143   match(Set dst (VectorStoreMask src size));
 8144   effect(TEMP_DEF dst, TEMP xtmp);
 8145   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
 8146   ins_encode %{
 8147     int vlen_enc = Assembler::AVX_128bit;
 8148     int vlen = Matcher::vector_length(this);
 8149     if (vlen <= 8) {
 8150       assert(UseSSE >= 3, "required");
 8151       __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
 8152       __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
 8153       __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
 8154     } else {
 8155       assert(UseAVX > 0, "required");
 8156       __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
 8157       __ vpacksswb($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
 8158       __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
 8159     }
 8160   %}
 8161   ins_pipe( pipe_slow );
 8162 %}
 8163 
 8164 instruct vstoreMask4B(vec dst, vec src, vec xtmp, immI_4 size) %{
 8165   predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
 8166   match(Set dst (VectorStoreMask src size));
 8167   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
 8168   effect(TEMP_DEF dst, TEMP xtmp);
 8169   ins_encode %{
 8170     int vlen_enc = Assembler::AVX_128bit;
 8171     int vlen = Matcher::vector_length(this);
 8172     if (vlen <= 4) {
 8173       assert(UseSSE >= 3, "required");
 8174       __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
 8175       __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
 8176       __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
 8177       __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
 8178     } else {
 8179       assert(UseAVX > 0, "required");
 8180       __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
 8181       __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
 8182       __ vpackssdw($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
 8183       __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
 8184       __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
 8185     }

 8205 
 8206 instruct storeMask8B_avx(vec dst, vec src, immI_8 size, vec vtmp) %{
 8207   predicate(UseAVX <= 2 && Matcher::vector_length(n) == 4);
 8208   match(Set dst (VectorStoreMask src size));
 8209   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s], using $vtmp as TEMP" %}
 8210   effect(TEMP_DEF dst, TEMP vtmp);
 8211   ins_encode %{
 8212     int vlen_enc = Assembler::AVX_128bit;
 8213     __ vshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 0x88, Assembler::AVX_256bit);
 8214     __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
 8215     __ vblendps($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0xC, vlen_enc);
 8216     __ vpxor($vtmp$$XMMRegister, $vtmp$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
 8217     __ vpackssdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
 8218     __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
 8219     __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
 8220   %}
 8221   ins_pipe( pipe_slow );
 8222 %}
 8223 
 8224 instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
 8225   predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
 8226   match(Set dst (VectorStoreMask src size));
 8227   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
 8228   ins_encode %{
 8229     int src_vlen_enc = vector_length_encoding(this, $src);
 8230     int dst_vlen_enc = vector_length_encoding(this);
 8231     if (!VM_Version::supports_avx512vl()) {
 8232       src_vlen_enc = Assembler::AVX_512bit;
 8233     }
 8234     __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
 8235     __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
 8236   %}
 8237   ins_pipe( pipe_slow );
 8238 %}
 8239 
 8240 instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
 8241   predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
 8242   match(Set dst (VectorStoreMask src size));
 8243   format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
 8244   ins_encode %{
 8245     int src_vlen_enc = vector_length_encoding(this, $src);
 8246     int dst_vlen_enc = vector_length_encoding(this);
 8247     if (!VM_Version::supports_avx512vl()) {
 8248       src_vlen_enc = Assembler::AVX_512bit;
 8249     }
 8250     __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
 8251     __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
 8252   %}
 8253   ins_pipe( pipe_slow );
 8254 %}
 8255 
 8256 instruct vstoreMask_evex_vectmask(vec dst, kReg mask, immI size) %{
 8257   predicate(n->in(1)->bottom_type()->isa_vectmask() && !VM_Version::supports_avx512vlbw());
 8258   match(Set dst (VectorStoreMask mask size));
 8259   effect(TEMP_DEF dst);
 8260   format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
 8261   ins_encode %{

 9043 %}
 9044 
 9045 instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
 9046   predicate(n->in(1)->bottom_type()->isa_vectmask());
 9047   match(Set dst (VectorMaskToLong mask));
 9048   effect(TEMP dst, KILL cr);
 9049   format %{ "vector_tolong_evex $dst, $mask \t! vector mask tolong" %}
 9050   ins_encode %{
 9051     int opcode = this->ideal_Opcode();
 9052     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9053     int mask_len = Matcher::vector_length(this, $mask);
 9054     int mask_size = mask_len * type2aelembytes(mbt);
 9055     int vlen_enc = vector_length_encoding(this, $mask);
 9056     __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
 9057                              $dst$$Register, mask_len, mask_size, vlen_enc);
 9058   %}
 9059   ins_pipe( pipe_slow );
 9060 %}
 9061 
 9062 instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
 9063   predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
 9064   match(Set dst (VectorMaskToLong mask));
 9065   format %{ "vector_tolong_bool $dst, $mask \t! using $xtmp as TEMP" %}
 9066   effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
 9067   ins_encode %{
 9068     int opcode = this->ideal_Opcode();
 9069     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9070     int mask_len = Matcher::vector_length(this, $mask);
 9071     int vlen_enc = vector_length_encoding(this, $mask);
 9072     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9073                              $dst$$Register, mask_len, mbt, vlen_enc);
 9074   %}
 9075   ins_pipe( pipe_slow );
 9076 %}
 9077 
 9078 instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr) %{
 9079   predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
 9080   match(Set dst (VectorMaskToLong (VectorStoreMask mask size)));
 9081   format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
 9082   effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
 9083   ins_encode %{
 9084     int opcode = this->ideal_Opcode();
 9085     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9086     int mask_len = Matcher::vector_length(this, $mask);
 9087     int vlen_enc = vector_length_encoding(this, $mask);
 9088     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9089                              $dst$$Register, mask_len, mbt, vlen_enc);
 9090   %}
 9091   ins_pipe( pipe_slow );
 9092 %}
 9093 
 9094 instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
 9095   predicate(n->in(1)->bottom_type()->isa_vectmask());
 9096   match(Set dst (VectorMaskTrueCount mask));
 9097   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
 9098   format %{ "vector_truecount_evex $dst, $mask \t! using $tmp as TEMP" %}
 9099   ins_encode %{
 9100     int opcode = this->ideal_Opcode();
 9101     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9102     int mask_len = Matcher::vector_length(this, $mask);
 9103     int mask_size = mask_len * type2aelembytes(mbt);
 9104     int vlen_enc = vector_length_encoding(this, $mask);
 9105     __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
 9106                              $tmp$$Register, mask_len, mask_size, vlen_enc);
 9107   %}
 9108   ins_pipe( pipe_slow );
 9109 %}
 9110 
 9111 instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
 9112   predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
 9113   match(Set dst (VectorMaskTrueCount mask));
 9114   effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
 9115   format %{ "vector_truecount_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
 9116   ins_encode %{
 9117     int opcode = this->ideal_Opcode();
 9118     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9119     int mask_len = Matcher::vector_length(this, $mask);
 9120     int vlen_enc = vector_length_encoding(this, $mask);
 9121     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9122                              $tmp$$Register, mask_len, mbt, vlen_enc);
 9123   %}
 9124   ins_pipe( pipe_slow );
 9125 %}
 9126 
 9127 instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
 9128   predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
 9129   match(Set dst (VectorMaskTrueCount (VectorStoreMask mask size)));
 9130   effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
 9131   format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
 9132   ins_encode %{
 9133     int opcode = this->ideal_Opcode();
 9134     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9135     int mask_len = Matcher::vector_length(this, $mask);
 9136     int vlen_enc = vector_length_encoding(this, $mask);
 9137     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9138                              $tmp$$Register, mask_len, mbt, vlen_enc);
 9139   %}
 9140   ins_pipe( pipe_slow );
 9141 %}
 9142 
 9143 instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
 9144   predicate(n->in(1)->bottom_type()->isa_vectmask());
 9145   match(Set dst (VectorMaskFirstTrue mask));
 9146   match(Set dst (VectorMaskLastTrue mask));
 9147   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
 9148   format %{ "vector_mask_first_or_last_true_evex $dst, $mask \t! using $tmp as TEMP" %}
 9149   ins_encode %{
 9150     int opcode = this->ideal_Opcode();
 9151     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9152     int mask_len = Matcher::vector_length(this, $mask);
 9153     int mask_size = mask_len * type2aelembytes(mbt);
 9154     int vlen_enc = vector_length_encoding(this, $mask);
 9155     __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
 9156                              $tmp$$Register, mask_len, mask_size, vlen_enc);
 9157   %}
 9158   ins_pipe( pipe_slow );
 9159 %}
 9160 
 9161 instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
 9162   predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
 9163   match(Set dst (VectorMaskFirstTrue mask));
 9164   match(Set dst (VectorMaskLastTrue mask));
 9165   effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
 9166   format %{ "vector_mask_first_or_last_true_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
 9167   ins_encode %{
 9168     int opcode = this->ideal_Opcode();
 9169     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9170     int mask_len = Matcher::vector_length(this, $mask);
 9171     int vlen_enc = vector_length_encoding(this, $mask);
 9172     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9173                              $tmp$$Register, mask_len, mbt, vlen_enc);
 9174   %}
 9175   ins_pipe( pipe_slow );
 9176 %}
 9177 
 9178 instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
 9179   predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
 9180   match(Set dst (VectorMaskFirstTrue (VectorStoreMask mask size)));
 9181   match(Set dst (VectorMaskLastTrue (VectorStoreMask mask size)));
 9182   effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
 9183   format %{ "vector_mask_first_or_last_true_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
 9184   ins_encode %{
 9185     int opcode = this->ideal_Opcode();
 9186     BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
 9187     int mask_len = Matcher::vector_length(this, $mask);
 9188     int vlen_enc = vector_length_encoding(this, $mask);
 9189     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
 9190                              $tmp$$Register, mask_len, mbt, vlen_enc);
 9191   %}
 9192   ins_pipe( pipe_slow );
 9193 %}
 9194 
 9195 // --------------------------------- Compress/Expand Operations ---------------------------
 9196 
 9197 instruct vcompress_expand_reg_evex(vec dst, vec src, kReg mask) %{
 9198   match(Set dst (CompressV src mask));
 9199   match(Set dst (ExpandV src mask));

 9994     uint masklen = Matcher::vector_length(this);
 9995     __ knot(masklen, $dst$$KRegister, $src$$KRegister, $ktmp$$KRegister, $rtmp$$Register);
 9996   %}
 9997   ins_pipe( pipe_slow );
 9998 %}
 9999 
10000 instruct mask_not_imm(kReg dst, kReg src, immI_M1 cnt) %{
10001   predicate((Matcher::vector_length(n) == 8 && VM_Version::supports_avx512dq()) ||
10002             (Matcher::vector_length(n) == 16) ||
10003             (Matcher::vector_length(n) > 16 && VM_Version::supports_avx512bw()));
10004   match(Set dst (XorVMask src (MaskAll cnt)));
10005   format %{ "mask_not $dst, $src, $cnt \t! mask not operation" %}
10006   ins_encode %{
10007     uint masklen = Matcher::vector_length(this);
10008     __ knot(masklen, $dst$$KRegister, $src$$KRegister);
10009   %}
10010   ins_pipe( pipe_slow );
10011 %}
10012 
10013 instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp) %{
10014   predicate(n->bottom_type()->isa_vectmask() == nullptr && Matcher::vector_length(n) <= 8);
10015   match(Set dst (VectorLongToMask src));
10016   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp);
10017   format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp as TEMP" %}
10018   ins_encode %{
10019     int mask_len = Matcher::vector_length(this);
10020     int vec_enc  = vector_length_encoding(mask_len);
10021     __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
10022                               $rtmp2$$Register, xnoreg, mask_len, vec_enc);
10023   %}
10024   ins_pipe( pipe_slow );
10025 %}
10026 
10027 
10028 instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp1, rFlagsReg cr) %{
10029   predicate(n->bottom_type()->isa_vectmask() == nullptr && Matcher::vector_length(n) > 8);
10030   match(Set dst (VectorLongToMask src));
10031   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp1, KILL cr);
10032   format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp1, as TEMP" %}
10033   ins_encode %{
10034     int mask_len = Matcher::vector_length(this);
10035     assert(mask_len <= 32, "invalid mask length");
10036     int vec_enc  = vector_length_encoding(mask_len);
10037     __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
10038                               $rtmp2$$Register, $xtmp1$$XMMRegister, mask_len, vec_enc);
10039   %}
10040   ins_pipe( pipe_slow );
10041 %}
10042 
10043 instruct long_to_mask_evex(kReg dst, rRegL src) %{
10044   predicate(n->bottom_type()->isa_vectmask());
10045   match(Set dst (VectorLongToMask src));
10046   format %{ "long_to_mask_evex $dst, $src\t!" %}
10047   ins_encode %{
10048     __ kmov($dst$$KRegister, $src$$Register);
10049   %}
< prev index next >