< prev index next >

src/hotspot/cpu/aarch64/aarch64.ad

Print this page

 2042 
 2043   return rc_stack;
 2044 }
 2045 
 2046 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2047   Compile* C = ra_->C;
 2048 
 2049   // Get registers to move.
 2050   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2051   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2052   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2053   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2054 
 2055   enum RC src_hi_rc = rc_class(src_hi);
 2056   enum RC src_lo_rc = rc_class(src_lo);
 2057   enum RC dst_hi_rc = rc_class(dst_hi);
 2058   enum RC dst_lo_rc = rc_class(dst_lo);
 2059 
 2060   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2061 
 2062   if (src_hi != OptoReg::Bad) {
 2063     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2064            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2065            "expected aligned-adjacent pairs");
 2066   }
 2067 
 2068   if (src_lo == dst_lo && src_hi == dst_hi) {
 2069     return 0;            // Self copy, no move.
 2070   }
 2071 
 2072   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2073               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2074   int src_offset = ra_->reg2offset(src_lo);
 2075   int dst_offset = ra_->reg2offset(dst_lo);
 2076 
 2077   if (bottom_type()->isa_vect() != NULL) {
 2078     uint ireg = ideal_reg();
 2079     if (ireg == Op_VecA && cbuf) {
 2080       C2_MacroAssembler _masm(cbuf);
 2081       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2082       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2083         // stack->stack
 2084         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2085                                                 sve_vector_reg_size_in_bytes);
 2086       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2087         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2088                             sve_vector_reg_size_in_bytes);
 2089       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2090         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2091                               sve_vector_reg_size_in_bytes);
 2092       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2093         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2094                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2095                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2096       } else {
 2097         ShouldNotReachHere();

 2163       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2164         if (is64) {
 2165             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2166                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2167         } else {
 2168             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2169                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2170         }
 2171       } else {                    // fpr --> stack spill
 2172         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2173         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2174                  is64 ? __ D : __ S, dst_offset);
 2175       }
 2176       break;
 2177     case rc_stack:
 2178       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2179         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2180       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2181         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2182                    is64 ? __ D : __ S, src_offset);



 2183       } else {                    // stack --> stack copy
 2184         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2185         __ unspill(rscratch1, is64, src_offset);
 2186         __ spill(rscratch1, is64, dst_offset);
















 2187       }
 2188       break;
 2189     default:
 2190       assert(false, "bad rc_class for spill");
 2191       ShouldNotReachHere();
 2192     }
 2193   }
 2194 
 2195   if (st) {
 2196     st->print("spill ");
 2197     if (src_lo_rc == rc_stack) {
 2198       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2199     } else {
 2200       st->print("%s -> ", Matcher::regName[src_lo]);
 2201     }
 2202     if (dst_lo_rc == rc_stack) {
 2203       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2204     } else {
 2205       st->print("%s", Matcher::regName[dst_lo]);
 2206     }
 2207     if (bottom_type()->isa_vect() != NULL) {
 2208       int vsize = 0;
 2209       switch (ideal_reg()) {
 2210       case Op_VecD:
 2211         vsize = 64;
 2212         break;
 2213       case Op_VecX:
 2214         vsize = 128;
 2215         break;
 2216       case Op_VecA:
 2217         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2218         break;
 2219       default:
 2220         assert(false, "bad register type for spill");
 2221         ShouldNotReachHere();
 2222       }
 2223       st->print("\t# vector spill size = %d", vsize);




 2224     } else {
 2225       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2226     }
 2227   }
 2228 
 2229   return 0;
 2230 
 2231 }
 2232 
 2233 #ifndef PRODUCT
 2234 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2235   if (!ra_)
 2236     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2237   else
 2238     implementation(NULL, ra_, false, st);
 2239 }
 2240 #endif
 2241 
 2242 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2243   implementation(&cbuf, ra_, false, NULL);

 2363   return offset;
 2364 }
 2365 
 2366 // REQUIRED MATCHER CODE
 2367 
 2368 //=============================================================================
 2369 
 2370 const bool Matcher::match_rule_supported(int opcode) {
 2371   if (!has_match_rule(opcode))
 2372     return false;
 2373 
 2374   bool ret_value = true;
 2375   switch (opcode) {
 2376     case Op_CacheWB:
 2377     case Op_CacheWBPreSync:
 2378     case Op_CacheWBPostSync:
 2379       if (!VM_Version::supports_data_cache_line_flush()) {
 2380         ret_value = false;
 2381       }
 2382       break;












 2383   }
 2384 
 2385   return ret_value; // Per default match rules are supported.
 2386 }
 2387 
 2388 // Identify extra cases that we might want to provide match rules for vector nodes and
 2389 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2390 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2391   if (!match_rule_supported(opcode)) {
 2392     return false;
 2393   }
 2394   int bit_size = vlen * type2aelembytes(bt) * 8;
 2395   if (UseSVE == 0 && bit_size > 128) {
 2396     return false;
 2397   }
 2398   if (UseSVE > 0) {
 2399     return op_sve_supported(opcode, vlen, bt);
 2400   } else { // NEON
 2401     // Special cases
 2402     switch (opcode) {

 2431       if (vlen < 4 || bit_size < 64) {
 2432         return false;
 2433       }
 2434       break;
 2435     case Op_VectorCastF2X:
 2436     case Op_VectorCastD2X:
 2437       if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
 2438         return false;
 2439       }
 2440       break;
 2441     case Op_LoadVectorGather:
 2442     case Op_StoreVectorScatter:
 2443       return false;
 2444     default:
 2445       break;
 2446     }
 2447   }
 2448   return vector_size_supported(bt, vlen);
 2449 }
 2450 









 2451 const RegMask* Matcher::predicate_reg_mask(void) {
 2452   return &_PR_REG_mask;
 2453 }
 2454 
 2455 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2456   return new TypeVectMask(elemTy, length);
 2457 }
 2458 
 2459 // Vector calling convention not yet implemented.
 2460 const bool Matcher::supports_vector_calling_convention(void) {
 2461   return false;
 2462 }
 2463 
 2464 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2465   Unimplemented();
 2466   return OptoRegPair(0, 0);
 2467 }
 2468 
 2469 // Is this branch offset short enough that a short branch can be used?
 2470 //

 2644     if (u->is_LoadStore()) {
 2645       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2646       // instructions) only take register indirect as an operand, so
 2647       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2648       // must fail.
 2649       return false;
 2650     }
 2651     if (u->is_Mem()) {
 2652       int opsize = u->as_Mem()->memory_size();
 2653       assert(opsize > 0, "unexpected memory operand size");
 2654       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2655         return false;
 2656       }
 2657     }
 2658   }
 2659   return true;
 2660 }
 2661 
 2662 // Should the matcher clone input 'm' of node 'n'?
 2663 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2664   if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
 2665     mstack.push(m, Visit);           // m = ShiftCntV



 2666     return true;
 2667   }

 2668   return false;
 2669 }
 2670 
 2671 // Should the Matcher clone shifts on addressing modes, expecting them
 2672 // to be subsumed into complex addressing expressions or compute them
 2673 // into registers?
 2674 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2675   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2676     return true;
 2677   }
 2678 
 2679   Node *off = m->in(AddPNode::Offset);
 2680   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2681       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2682       // Are there other uses besides address expressions?
 2683       !is_visited(off)) {
 2684     address_visited.set(off->_idx); // Flag as address_visited
 2685     mstack.push(off->in(2), Visit);
 2686     Node *conv = off->in(1);
 2687     if (conv->Opcode() == Op_ConvI2L &&

 5506   constraint(ALLOC_IN_RC(v30_reg));
 5507   match(RegD);
 5508   op_cost(0);
 5509   format %{ %}
 5510   interface(REG_INTER);
 5511 %}
 5512 
 5513 operand vRegD_V31()
 5514 %{
 5515   constraint(ALLOC_IN_RC(v31_reg));
 5516   match(RegD);
 5517   op_cost(0);
 5518   format %{ %}
 5519   interface(REG_INTER);
 5520 %}
 5521 
 5522 operand pReg()
 5523 %{
 5524   constraint(ALLOC_IN_RC(pr_reg));
 5525   match(RegVectMask);

 5526   op_cost(0);
 5527   format %{ %}
 5528   interface(REG_INTER);
 5529 %}
 5530 
 5531 operand pRegGov()
 5532 %{
 5533   constraint(ALLOC_IN_RC(gov_pr));
 5534   match(RegVectMask);
 5535   op_cost(0);
 5536   format %{ %}
 5537   interface(REG_INTER);
 5538 %}
 5539 
 5540 // Flags register, used as output of signed compare instructions
 5541 
 5542 // note that on AArch64 we also use this register as the output for
 5543 // for floating point compare instructions (CmpF CmpD). this ensures
 5544 // that ordered inequality tests use GT, GE, LT or LE none of which
 5545 // pass through cases where the result is unordered i.e. one or both

 8854   match(Set dst (CastVV dst));
 8855 
 8856   size(0);
 8857   format %{ "# castVV of $dst" %}
 8858   ins_encode(/* empty encoding */);
 8859   ins_cost(0);
 8860   ins_pipe(pipe_class_empty);
 8861 %}
 8862 
 8863 instruct castVV(vReg dst)
 8864 %{
 8865   match(Set dst (CastVV dst));
 8866 
 8867   size(0);
 8868   format %{ "# castVV of $dst" %}
 8869   ins_encode(/* empty encoding */);
 8870   ins_cost(0);
 8871   ins_pipe(pipe_class_empty);
 8872 %}
 8873 











 8874 // ============================================================================
 8875 // Atomic operation instructions
 8876 //
 8877 // Intel and SPARC both implement Ideal Node LoadPLocked and
 8878 // Store{PIL}Conditional instructions using a normal load for the
 8879 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 8880 //
 8881 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 8882 // pair to lock object allocations from Eden space when not using
 8883 // TLABs.
 8884 //
 8885 // There does not appear to be a Load{IL}Locked Ideal Node and the
 8886 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 8887 // and to use StoreIConditional only for 32-bit and StoreLConditional
 8888 // only for 64-bit.
 8889 //
 8890 // We implement LoadPLocked and StorePLocked instructions using,
 8891 // respectively the AArch64 hw load-exclusive and store-conditional
 8892 // instructions. Whereas we must implement each of
 8893 // Store{IL}Conditional using a CAS which employs a pair of

 2042 
 2043   return rc_stack;
 2044 }
 2045 
 2046 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2047   Compile* C = ra_->C;
 2048 
 2049   // Get registers to move.
 2050   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2051   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2052   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2053   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2054 
 2055   enum RC src_hi_rc = rc_class(src_hi);
 2056   enum RC src_lo_rc = rc_class(src_lo);
 2057   enum RC dst_hi_rc = rc_class(dst_hi);
 2058   enum RC dst_lo_rc = rc_class(dst_lo);
 2059 
 2060   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2061 
 2062   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 2063     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2064            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2065            "expected aligned-adjacent pairs");
 2066   }
 2067 
 2068   if (src_lo == dst_lo && src_hi == dst_hi) {
 2069     return 0;            // Self copy, no move.
 2070   }
 2071 
 2072   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2073               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2074   int src_offset = ra_->reg2offset(src_lo);
 2075   int dst_offset = ra_->reg2offset(dst_lo);
 2076 
 2077   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2078     uint ireg = ideal_reg();
 2079     if (ireg == Op_VecA && cbuf) {
 2080       C2_MacroAssembler _masm(cbuf);
 2081       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2082       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2083         // stack->stack
 2084         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2085                                                 sve_vector_reg_size_in_bytes);
 2086       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2087         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2088                             sve_vector_reg_size_in_bytes);
 2089       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2090         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2091                               sve_vector_reg_size_in_bytes);
 2092       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2093         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2094                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2095                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2096       } else {
 2097         ShouldNotReachHere();

 2163       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2164         if (is64) {
 2165             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2166                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2167         } else {
 2168             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2169                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2170         }
 2171       } else {                    // fpr --> stack spill
 2172         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2173         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2174                  is64 ? __ D : __ S, dst_offset);
 2175       }
 2176       break;
 2177     case rc_stack:
 2178       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2179         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2180       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2181         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2182                    is64 ? __ D : __ S, src_offset);
 2183       } else if (dst_lo_rc == rc_predicate) {
 2184         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2185                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2186       } else {                    // stack --> stack copy
 2187         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2188         if (ideal_reg() == Op_RegVectMask) {
 2189           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2190                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2191         } else {
 2192           __ unspill(rscratch1, is64, src_offset);
 2193           __ spill(rscratch1, is64, dst_offset);
 2194         }
 2195       }
 2196       break;
 2197     case rc_predicate:
 2198       if (dst_lo_rc == rc_predicate) {
 2199         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2200       } else if (dst_lo_rc == rc_stack) {
 2201         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2202                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2203       } else {
 2204         assert(false, "bad src and dst rc_class combination.");
 2205         ShouldNotReachHere();
 2206       }
 2207       break;
 2208     default:
 2209       assert(false, "bad rc_class for spill");
 2210       ShouldNotReachHere();
 2211     }
 2212   }
 2213 
 2214   if (st) {
 2215     st->print("spill ");
 2216     if (src_lo_rc == rc_stack) {
 2217       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2218     } else {
 2219       st->print("%s -> ", Matcher::regName[src_lo]);
 2220     }
 2221     if (dst_lo_rc == rc_stack) {
 2222       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2223     } else {
 2224       st->print("%s", Matcher::regName[dst_lo]);
 2225     }
 2226     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2227       int vsize = 0;
 2228       switch (ideal_reg()) {
 2229       case Op_VecD:
 2230         vsize = 64;
 2231         break;
 2232       case Op_VecX:
 2233         vsize = 128;
 2234         break;
 2235       case Op_VecA:
 2236         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2237         break;
 2238       default:
 2239         assert(false, "bad register type for spill");
 2240         ShouldNotReachHere();
 2241       }
 2242       st->print("\t# vector spill size = %d", vsize);
 2243     } else if (ideal_reg() == Op_RegVectMask) {
 2244       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2245       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2246       st->print("\t# predicate spill size = %d", vsize);
 2247     } else {
 2248       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2249     }
 2250   }
 2251 
 2252   return 0;
 2253 
 2254 }
 2255 
 2256 #ifndef PRODUCT
 2257 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2258   if (!ra_)
 2259     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2260   else
 2261     implementation(NULL, ra_, false, st);
 2262 }
 2263 #endif
 2264 
 2265 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2266   implementation(&cbuf, ra_, false, NULL);

 2386   return offset;
 2387 }
 2388 
 2389 // REQUIRED MATCHER CODE
 2390 
 2391 //=============================================================================
 2392 
 2393 const bool Matcher::match_rule_supported(int opcode) {
 2394   if (!has_match_rule(opcode))
 2395     return false;
 2396 
 2397   bool ret_value = true;
 2398   switch (opcode) {
 2399     case Op_CacheWB:
 2400     case Op_CacheWBPreSync:
 2401     case Op_CacheWBPostSync:
 2402       if (!VM_Version::supports_data_cache_line_flush()) {
 2403         ret_value = false;
 2404       }
 2405       break;
 2406     case Op_LoadVectorMasked:
 2407     case Op_StoreVectorMasked:
 2408     case Op_LoadVectorGatherMasked:
 2409     case Op_StoreVectorScatterMasked:
 2410     case Op_MaskAll:
 2411     case Op_AndVMask:
 2412     case Op_OrVMask:
 2413     case Op_XorVMask:
 2414       if (UseSVE == 0) {
 2415         ret_value = false;
 2416       }
 2417       break;
 2418   }
 2419 
 2420   return ret_value; // Per default match rules are supported.
 2421 }
 2422 
 2423 // Identify extra cases that we might want to provide match rules for vector nodes and
 2424 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2425 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2426   if (!match_rule_supported(opcode)) {
 2427     return false;
 2428   }
 2429   int bit_size = vlen * type2aelembytes(bt) * 8;
 2430   if (UseSVE == 0 && bit_size > 128) {
 2431     return false;
 2432   }
 2433   if (UseSVE > 0) {
 2434     return op_sve_supported(opcode, vlen, bt);
 2435   } else { // NEON
 2436     // Special cases
 2437     switch (opcode) {

 2466       if (vlen < 4 || bit_size < 64) {
 2467         return false;
 2468       }
 2469       break;
 2470     case Op_VectorCastF2X:
 2471     case Op_VectorCastD2X:
 2472       if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
 2473         return false;
 2474       }
 2475       break;
 2476     case Op_LoadVectorGather:
 2477     case Op_StoreVectorScatter:
 2478       return false;
 2479     default:
 2480       break;
 2481     }
 2482   }
 2483   return vector_size_supported(bt, vlen);
 2484 }
 2485 
 2486 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
 2487   // Only SVE supports masked operations.
 2488   if (UseSVE == 0) {
 2489     return false;
 2490   }
 2491   return match_rule_supported(opcode) &&
 2492          masked_op_sve_supported(opcode, vlen, bt);
 2493 }
 2494 
 2495 const RegMask* Matcher::predicate_reg_mask(void) {
 2496   return &_PR_REG_mask;
 2497 }
 2498 
 2499 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2500   return new TypeVectMask(elemTy, length);
 2501 }
 2502 
 2503 // Vector calling convention not yet implemented.
 2504 const bool Matcher::supports_vector_calling_convention(void) {
 2505   return false;
 2506 }
 2507 
 2508 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2509   Unimplemented();
 2510   return OptoRegPair(0, 0);
 2511 }
 2512 
 2513 // Is this branch offset short enough that a short branch can be used?
 2514 //

 2688     if (u->is_LoadStore()) {
 2689       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2690       // instructions) only take register indirect as an operand, so
 2691       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2692       // must fail.
 2693       return false;
 2694     }
 2695     if (u->is_Mem()) {
 2696       int opsize = u->as_Mem()->memory_size();
 2697       assert(opsize > 0, "unexpected memory operand size");
 2698       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2699         return false;
 2700       }
 2701     }
 2702   }
 2703   return true;
 2704 }
 2705 
 2706 // Should the matcher clone input 'm' of node 'n'?
 2707 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2708   // ShiftV src (ShiftCntV con)
 2709   // StoreVector (VectorStoreMask src)
 2710   if (is_vshift_con_pattern(n, m) ||
 2711       (UseSVE > 0 && m->Opcode() == Op_VectorStoreMask && n->Opcode() == Op_StoreVector)) {
 2712     mstack.push(m, Visit);
 2713     return true;
 2714   }
 2715 
 2716   return false;
 2717 }
 2718 
 2719 // Should the Matcher clone shifts on addressing modes, expecting them
 2720 // to be subsumed into complex addressing expressions or compute them
 2721 // into registers?
 2722 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2723   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2724     return true;
 2725   }
 2726 
 2727   Node *off = m->in(AddPNode::Offset);
 2728   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2729       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2730       // Are there other uses besides address expressions?
 2731       !is_visited(off)) {
 2732     address_visited.set(off->_idx); // Flag as address_visited
 2733     mstack.push(off->in(2), Visit);
 2734     Node *conv = off->in(1);
 2735     if (conv->Opcode() == Op_ConvI2L &&

 5554   constraint(ALLOC_IN_RC(v30_reg));
 5555   match(RegD);
 5556   op_cost(0);
 5557   format %{ %}
 5558   interface(REG_INTER);
 5559 %}
 5560 
 5561 operand vRegD_V31()
 5562 %{
 5563   constraint(ALLOC_IN_RC(v31_reg));
 5564   match(RegD);
 5565   op_cost(0);
 5566   format %{ %}
 5567   interface(REG_INTER);
 5568 %}
 5569 
 5570 operand pReg()
 5571 %{
 5572   constraint(ALLOC_IN_RC(pr_reg));
 5573   match(RegVectMask);
 5574   match(pRegGov);
 5575   op_cost(0);
 5576   format %{ %}
 5577   interface(REG_INTER);
 5578 %}
 5579 
 5580 operand pRegGov()
 5581 %{
 5582   constraint(ALLOC_IN_RC(gov_pr));
 5583   match(RegVectMask);
 5584   op_cost(0);
 5585   format %{ %}
 5586   interface(REG_INTER);
 5587 %}
 5588 
 5589 // Flags register, used as output of signed compare instructions
 5590 
 5591 // note that on AArch64 we also use this register as the output for
 5592 // for floating point compare instructions (CmpF CmpD). this ensures
 5593 // that ordered inequality tests use GT, GE, LT or LE none of which
 5594 // pass through cases where the result is unordered i.e. one or both

 8903   match(Set dst (CastVV dst));
 8904 
 8905   size(0);
 8906   format %{ "# castVV of $dst" %}
 8907   ins_encode(/* empty encoding */);
 8908   ins_cost(0);
 8909   ins_pipe(pipe_class_empty);
 8910 %}
 8911 
 8912 instruct castVV(vReg dst)
 8913 %{
 8914   match(Set dst (CastVV dst));
 8915 
 8916   size(0);
 8917   format %{ "# castVV of $dst" %}
 8918   ins_encode(/* empty encoding */);
 8919   ins_cost(0);
 8920   ins_pipe(pipe_class_empty);
 8921 %}
 8922 
 8923 instruct castVVMask(pRegGov dst)
 8924 %{
 8925   match(Set dst (CastVV dst));
 8926 
 8927   size(0);
 8928   format %{ "# castVV of $dst" %}
 8929   ins_encode(/* empty encoding */);
 8930   ins_cost(0);
 8931   ins_pipe(pipe_class_empty);
 8932 %}
 8933 
 8934 // ============================================================================
 8935 // Atomic operation instructions
 8936 //
 8937 // Intel and SPARC both implement Ideal Node LoadPLocked and
 8938 // Store{PIL}Conditional instructions using a normal load for the
 8939 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 8940 //
 8941 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 8942 // pair to lock object allocations from Eden space when not using
 8943 // TLABs.
 8944 //
 8945 // There does not appear to be a Load{IL}Locked Ideal Node and the
 8946 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 8947 // and to use StoreIConditional only for 32-bit and StoreLConditional
 8948 // only for 64-bit.
 8949 //
 8950 // We implement LoadPLocked and StorePLocked instructions using,
 8951 // respectively the AArch64 hw load-exclusive and store-conditional
 8952 // instructions. Whereas we must implement each of
 8953 // Store{IL}Conditional using a CAS which employs a pair of
< prev index next >