2042
2043 return rc_stack;
2044 }
2045
2046 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
2047 Compile* C = ra_->C;
2048
2049 // Get registers to move.
2050 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
2051 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
2052 OptoReg::Name dst_hi = ra_->get_reg_second(this);
2053 OptoReg::Name dst_lo = ra_->get_reg_first(this);
2054
2055 enum RC src_hi_rc = rc_class(src_hi);
2056 enum RC src_lo_rc = rc_class(src_lo);
2057 enum RC dst_hi_rc = rc_class(dst_hi);
2058 enum RC dst_lo_rc = rc_class(dst_lo);
2059
2060 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
2061
2062 if (src_hi != OptoReg::Bad) {
2063 assert((src_lo&1)==0 && src_lo+1==src_hi &&
2064 (dst_lo&1)==0 && dst_lo+1==dst_hi,
2065 "expected aligned-adjacent pairs");
2066 }
2067
2068 if (src_lo == dst_lo && src_hi == dst_hi) {
2069 return 0; // Self copy, no move.
2070 }
2071
2072 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2073 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2074 int src_offset = ra_->reg2offset(src_lo);
2075 int dst_offset = ra_->reg2offset(dst_lo);
2076
2077 if (bottom_type()->isa_vect() != NULL) {
2078 uint ireg = ideal_reg();
2079 if (ireg == Op_VecA && cbuf) {
2080 C2_MacroAssembler _masm(cbuf);
2081 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2082 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2083 // stack->stack
2084 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2085 sve_vector_reg_size_in_bytes);
2086 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2087 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2088 sve_vector_reg_size_in_bytes);
2089 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2090 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2091 sve_vector_reg_size_in_bytes);
2092 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2093 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2094 as_FloatRegister(Matcher::_regEncode[src_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 } else {
2097 ShouldNotReachHere();
2163 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2164 if (is64) {
2165 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2166 as_FloatRegister(Matcher::_regEncode[src_lo]));
2167 } else {
2168 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2169 as_FloatRegister(Matcher::_regEncode[src_lo]));
2170 }
2171 } else { // fpr --> stack spill
2172 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2173 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2174 is64 ? __ D : __ S, dst_offset);
2175 }
2176 break;
2177 case rc_stack:
2178 if (dst_lo_rc == rc_int) { // stack --> gpr load
2179 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2180 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2181 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2182 is64 ? __ D : __ S, src_offset);
2183 } else { // stack --> stack copy
2184 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2185 __ unspill(rscratch1, is64, src_offset);
2186 __ spill(rscratch1, is64, dst_offset);
2187 }
2188 break;
2189 default:
2190 assert(false, "bad rc_class for spill");
2191 ShouldNotReachHere();
2192 }
2193 }
2194
2195 if (st) {
2196 st->print("spill ");
2197 if (src_lo_rc == rc_stack) {
2198 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2199 } else {
2200 st->print("%s -> ", Matcher::regName[src_lo]);
2201 }
2202 if (dst_lo_rc == rc_stack) {
2203 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2204 } else {
2205 st->print("%s", Matcher::regName[dst_lo]);
2206 }
2207 if (bottom_type()->isa_vect() != NULL) {
2208 int vsize = 0;
2209 switch (ideal_reg()) {
2210 case Op_VecD:
2211 vsize = 64;
2212 break;
2213 case Op_VecX:
2214 vsize = 128;
2215 break;
2216 case Op_VecA:
2217 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2218 break;
2219 default:
2220 assert(false, "bad register type for spill");
2221 ShouldNotReachHere();
2222 }
2223 st->print("\t# vector spill size = %d", vsize);
2224 } else {
2225 st->print("\t# spill size = %d", is64 ? 64 : 32);
2226 }
2227 }
2228
2229 return 0;
2230
2231 }
2232
2233 #ifndef PRODUCT
2234 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2235 if (!ra_)
2236 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2237 else
2238 implementation(NULL, ra_, false, st);
2239 }
2240 #endif
2241
2242 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2243 implementation(&cbuf, ra_, false, NULL);
2363 return offset;
2364 }
2365
2366 // REQUIRED MATCHER CODE
2367
2368 //=============================================================================
2369
2370 const bool Matcher::match_rule_supported(int opcode) {
2371 if (!has_match_rule(opcode))
2372 return false;
2373
2374 bool ret_value = true;
2375 switch (opcode) {
2376 case Op_CacheWB:
2377 case Op_CacheWBPreSync:
2378 case Op_CacheWBPostSync:
2379 if (!VM_Version::supports_data_cache_line_flush()) {
2380 ret_value = false;
2381 }
2382 break;
2383 }
2384
2385 return ret_value; // Per default match rules are supported.
2386 }
2387
2388 // Identify extra cases that we might want to provide match rules for vector nodes and
2389 // other intrinsics guarded with vector length (vlen) and element type (bt).
2390 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
2391 if (!match_rule_supported(opcode)) {
2392 return false;
2393 }
2394 int bit_size = vlen * type2aelembytes(bt) * 8;
2395 if (UseSVE == 0 && bit_size > 128) {
2396 return false;
2397 }
2398 if (UseSVE > 0) {
2399 return op_sve_supported(opcode, vlen, bt);
2400 } else { // NEON
2401 // Special cases
2402 switch (opcode) {
2431 if (vlen < 4 || bit_size < 64) {
2432 return false;
2433 }
2434 break;
2435 case Op_VectorCastF2X:
2436 case Op_VectorCastD2X:
2437 if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
2438 return false;
2439 }
2440 break;
2441 case Op_LoadVectorGather:
2442 case Op_StoreVectorScatter:
2443 return false;
2444 default:
2445 break;
2446 }
2447 }
2448 return vector_size_supported(bt, vlen);
2449 }
2450
2451 const RegMask* Matcher::predicate_reg_mask(void) {
2452 return &_PR_REG_mask;
2453 }
2454
2455 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2456 return new TypeVectMask(elemTy, length);
2457 }
2458
2459 // Vector calling convention not yet implemented.
2460 const bool Matcher::supports_vector_calling_convention(void) {
2461 return false;
2462 }
2463
2464 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2465 Unimplemented();
2466 return OptoRegPair(0, 0);
2467 }
2468
2469 // Is this branch offset short enough that a short branch can be used?
2470 //
2644 if (u->is_LoadStore()) {
2645 // On AArch64, LoadStoreNodes (i.e. compare and swap
2646 // instructions) only take register indirect as an operand, so
2647 // any attempt to use an AddPNode as an input to a LoadStoreNode
2648 // must fail.
2649 return false;
2650 }
2651 if (u->is_Mem()) {
2652 int opsize = u->as_Mem()->memory_size();
2653 assert(opsize > 0, "unexpected memory operand size");
2654 if (u->as_Mem()->memory_size() != (1<<shift)) {
2655 return false;
2656 }
2657 }
2658 }
2659 return true;
2660 }
2661
2662 // Should the matcher clone input 'm' of node 'n'?
2663 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2664 if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
2665 mstack.push(m, Visit); // m = ShiftCntV
2666 return true;
2667 }
2668 return false;
2669 }
2670
2671 // Should the Matcher clone shifts on addressing modes, expecting them
2672 // to be subsumed into complex addressing expressions or compute them
2673 // into registers?
2674 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2675 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2676 return true;
2677 }
2678
2679 Node *off = m->in(AddPNode::Offset);
2680 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2681 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2682 // Are there other uses besides address expressions?
2683 !is_visited(off)) {
2684 address_visited.set(off->_idx); // Flag as address_visited
2685 mstack.push(off->in(2), Visit);
2686 Node *conv = off->in(1);
2687 if (conv->Opcode() == Op_ConvI2L &&
5506 constraint(ALLOC_IN_RC(v30_reg));
5507 match(RegD);
5508 op_cost(0);
5509 format %{ %}
5510 interface(REG_INTER);
5511 %}
5512
5513 operand vRegD_V31()
5514 %{
5515 constraint(ALLOC_IN_RC(v31_reg));
5516 match(RegD);
5517 op_cost(0);
5518 format %{ %}
5519 interface(REG_INTER);
5520 %}
5521
5522 operand pReg()
5523 %{
5524 constraint(ALLOC_IN_RC(pr_reg));
5525 match(RegVectMask);
5526 op_cost(0);
5527 format %{ %}
5528 interface(REG_INTER);
5529 %}
5530
5531 operand pRegGov()
5532 %{
5533 constraint(ALLOC_IN_RC(gov_pr));
5534 match(RegVectMask);
5535 op_cost(0);
5536 format %{ %}
5537 interface(REG_INTER);
5538 %}
5539
5540 // Flags register, used as output of signed compare instructions
5541
5542 // note that on AArch64 we also use this register as the output for
5543 // for floating point compare instructions (CmpF CmpD). this ensures
5544 // that ordered inequality tests use GT, GE, LT or LE none of which
5545 // pass through cases where the result is unordered i.e. one or both
8854 match(Set dst (CastVV dst));
8855
8856 size(0);
8857 format %{ "# castVV of $dst" %}
8858 ins_encode(/* empty encoding */);
8859 ins_cost(0);
8860 ins_pipe(pipe_class_empty);
8861 %}
8862
8863 instruct castVV(vReg dst)
8864 %{
8865 match(Set dst (CastVV dst));
8866
8867 size(0);
8868 format %{ "# castVV of $dst" %}
8869 ins_encode(/* empty encoding */);
8870 ins_cost(0);
8871 ins_pipe(pipe_class_empty);
8872 %}
8873
8874 // ============================================================================
8875 // Atomic operation instructions
8876 //
8877 // Intel and SPARC both implement Ideal Node LoadPLocked and
8878 // Store{PIL}Conditional instructions using a normal load for the
8879 // LoadPLocked and a CAS for the Store{PIL}Conditional.
8880 //
8881 // The ideal code appears only to use LoadPLocked/StorePLocked as a
8882 // pair to lock object allocations from Eden space when not using
8883 // TLABs.
8884 //
8885 // There does not appear to be a Load{IL}Locked Ideal Node and the
8886 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
8887 // and to use StoreIConditional only for 32-bit and StoreLConditional
8888 // only for 64-bit.
8889 //
8890 // We implement LoadPLocked and StorePLocked instructions using,
8891 // respectively the AArch64 hw load-exclusive and store-conditional
8892 // instructions. Whereas we must implement each of
8893 // Store{IL}Conditional using a CAS which employs a pair of
|
2042
2043 return rc_stack;
2044 }
2045
2046 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
2047 Compile* C = ra_->C;
2048
2049 // Get registers to move.
2050 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
2051 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
2052 OptoReg::Name dst_hi = ra_->get_reg_second(this);
2053 OptoReg::Name dst_lo = ra_->get_reg_first(this);
2054
2055 enum RC src_hi_rc = rc_class(src_hi);
2056 enum RC src_lo_rc = rc_class(src_lo);
2057 enum RC dst_hi_rc = rc_class(dst_hi);
2058 enum RC dst_lo_rc = rc_class(dst_lo);
2059
2060 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
2061
2062 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
2063 assert((src_lo&1)==0 && src_lo+1==src_hi &&
2064 (dst_lo&1)==0 && dst_lo+1==dst_hi,
2065 "expected aligned-adjacent pairs");
2066 }
2067
2068 if (src_lo == dst_lo && src_hi == dst_hi) {
2069 return 0; // Self copy, no move.
2070 }
2071
2072 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2073 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2074 int src_offset = ra_->reg2offset(src_lo);
2075 int dst_offset = ra_->reg2offset(dst_lo);
2076
2077 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2078 uint ireg = ideal_reg();
2079 if (ireg == Op_VecA && cbuf) {
2080 C2_MacroAssembler _masm(cbuf);
2081 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2082 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2083 // stack->stack
2084 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2085 sve_vector_reg_size_in_bytes);
2086 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2087 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2088 sve_vector_reg_size_in_bytes);
2089 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2090 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2091 sve_vector_reg_size_in_bytes);
2092 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2093 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2094 as_FloatRegister(Matcher::_regEncode[src_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 } else {
2097 ShouldNotReachHere();
2163 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2164 if (is64) {
2165 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2166 as_FloatRegister(Matcher::_regEncode[src_lo]));
2167 } else {
2168 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2169 as_FloatRegister(Matcher::_regEncode[src_lo]));
2170 }
2171 } else { // fpr --> stack spill
2172 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2173 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2174 is64 ? __ D : __ S, dst_offset);
2175 }
2176 break;
2177 case rc_stack:
2178 if (dst_lo_rc == rc_int) { // stack --> gpr load
2179 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2180 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2181 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2182 is64 ? __ D : __ S, src_offset);
2183 } else if (dst_lo_rc == rc_predicate) {
2184 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2185 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2186 } else { // stack --> stack copy
2187 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2188 if (ideal_reg() == Op_RegVectMask) {
2189 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2190 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2191 } else {
2192 __ unspill(rscratch1, is64, src_offset);
2193 __ spill(rscratch1, is64, dst_offset);
2194 }
2195 }
2196 break;
2197 case rc_predicate:
2198 if (dst_lo_rc == rc_predicate) {
2199 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2200 } else if (dst_lo_rc == rc_stack) {
2201 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2202 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2203 } else {
2204 assert(false, "bad src and dst rc_class combination.");
2205 ShouldNotReachHere();
2206 }
2207 break;
2208 default:
2209 assert(false, "bad rc_class for spill");
2210 ShouldNotReachHere();
2211 }
2212 }
2213
2214 if (st) {
2215 st->print("spill ");
2216 if (src_lo_rc == rc_stack) {
2217 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2218 } else {
2219 st->print("%s -> ", Matcher::regName[src_lo]);
2220 }
2221 if (dst_lo_rc == rc_stack) {
2222 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2223 } else {
2224 st->print("%s", Matcher::regName[dst_lo]);
2225 }
2226 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2227 int vsize = 0;
2228 switch (ideal_reg()) {
2229 case Op_VecD:
2230 vsize = 64;
2231 break;
2232 case Op_VecX:
2233 vsize = 128;
2234 break;
2235 case Op_VecA:
2236 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2237 break;
2238 default:
2239 assert(false, "bad register type for spill");
2240 ShouldNotReachHere();
2241 }
2242 st->print("\t# vector spill size = %d", vsize);
2243 } else if (ideal_reg() == Op_RegVectMask) {
2244 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2245 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2246 st->print("\t# predicate spill size = %d", vsize);
2247 } else {
2248 st->print("\t# spill size = %d", is64 ? 64 : 32);
2249 }
2250 }
2251
2252 return 0;
2253
2254 }
2255
2256 #ifndef PRODUCT
2257 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2258 if (!ra_)
2259 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2260 else
2261 implementation(NULL, ra_, false, st);
2262 }
2263 #endif
2264
2265 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2266 implementation(&cbuf, ra_, false, NULL);
2386 return offset;
2387 }
2388
2389 // REQUIRED MATCHER CODE
2390
2391 //=============================================================================
2392
2393 const bool Matcher::match_rule_supported(int opcode) {
2394 if (!has_match_rule(opcode))
2395 return false;
2396
2397 bool ret_value = true;
2398 switch (opcode) {
2399 case Op_CacheWB:
2400 case Op_CacheWBPreSync:
2401 case Op_CacheWBPostSync:
2402 if (!VM_Version::supports_data_cache_line_flush()) {
2403 ret_value = false;
2404 }
2405 break;
2406 case Op_LoadVectorMasked:
2407 case Op_StoreVectorMasked:
2408 case Op_LoadVectorGatherMasked:
2409 case Op_StoreVectorScatterMasked:
2410 case Op_MaskAll:
2411 case Op_AndVMask:
2412 case Op_OrVMask:
2413 case Op_XorVMask:
2414 if (UseSVE == 0) {
2415 ret_value = false;
2416 }
2417 break;
2418 }
2419
2420 return ret_value; // Per default match rules are supported.
2421 }
2422
2423 // Identify extra cases that we might want to provide match rules for vector nodes and
2424 // other intrinsics guarded with vector length (vlen) and element type (bt).
2425 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
2426 if (!match_rule_supported(opcode)) {
2427 return false;
2428 }
2429 int bit_size = vlen * type2aelembytes(bt) * 8;
2430 if (UseSVE == 0 && bit_size > 128) {
2431 return false;
2432 }
2433 if (UseSVE > 0) {
2434 return op_sve_supported(opcode, vlen, bt);
2435 } else { // NEON
2436 // Special cases
2437 switch (opcode) {
2466 if (vlen < 4 || bit_size < 64) {
2467 return false;
2468 }
2469 break;
2470 case Op_VectorCastF2X:
2471 case Op_VectorCastD2X:
2472 if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
2473 return false;
2474 }
2475 break;
2476 case Op_LoadVectorGather:
2477 case Op_StoreVectorScatter:
2478 return false;
2479 default:
2480 break;
2481 }
2482 }
2483 return vector_size_supported(bt, vlen);
2484 }
2485
2486 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
2487 // Only SVE supports masked operations.
2488 if (UseSVE == 0) {
2489 return false;
2490 }
2491 return match_rule_supported(opcode) &&
2492 masked_op_sve_supported(opcode, vlen, bt);
2493 }
2494
2495 const RegMask* Matcher::predicate_reg_mask(void) {
2496 return &_PR_REG_mask;
2497 }
2498
2499 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2500 return new TypeVectMask(elemTy, length);
2501 }
2502
2503 // Vector calling convention not yet implemented.
2504 const bool Matcher::supports_vector_calling_convention(void) {
2505 return false;
2506 }
2507
2508 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2509 Unimplemented();
2510 return OptoRegPair(0, 0);
2511 }
2512
2513 // Is this branch offset short enough that a short branch can be used?
2514 //
2688 if (u->is_LoadStore()) {
2689 // On AArch64, LoadStoreNodes (i.e. compare and swap
2690 // instructions) only take register indirect as an operand, so
2691 // any attempt to use an AddPNode as an input to a LoadStoreNode
2692 // must fail.
2693 return false;
2694 }
2695 if (u->is_Mem()) {
2696 int opsize = u->as_Mem()->memory_size();
2697 assert(opsize > 0, "unexpected memory operand size");
2698 if (u->as_Mem()->memory_size() != (1<<shift)) {
2699 return false;
2700 }
2701 }
2702 }
2703 return true;
2704 }
2705
2706 // Should the matcher clone input 'm' of node 'n'?
2707 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2708 // ShiftV src (ShiftCntV con)
2709 // StoreVector (VectorStoreMask src)
2710 if (is_vshift_con_pattern(n, m) ||
2711 (UseSVE > 0 && m->Opcode() == Op_VectorStoreMask && n->Opcode() == Op_StoreVector)) {
2712 mstack.push(m, Visit);
2713 return true;
2714 }
2715
2716 return false;
2717 }
2718
2719 // Should the Matcher clone shifts on addressing modes, expecting them
2720 // to be subsumed into complex addressing expressions or compute them
2721 // into registers?
2722 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2723 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2724 return true;
2725 }
2726
2727 Node *off = m->in(AddPNode::Offset);
2728 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2729 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2730 // Are there other uses besides address expressions?
2731 !is_visited(off)) {
2732 address_visited.set(off->_idx); // Flag as address_visited
2733 mstack.push(off->in(2), Visit);
2734 Node *conv = off->in(1);
2735 if (conv->Opcode() == Op_ConvI2L &&
5554 constraint(ALLOC_IN_RC(v30_reg));
5555 match(RegD);
5556 op_cost(0);
5557 format %{ %}
5558 interface(REG_INTER);
5559 %}
5560
5561 operand vRegD_V31()
5562 %{
5563 constraint(ALLOC_IN_RC(v31_reg));
5564 match(RegD);
5565 op_cost(0);
5566 format %{ %}
5567 interface(REG_INTER);
5568 %}
5569
5570 operand pReg()
5571 %{
5572 constraint(ALLOC_IN_RC(pr_reg));
5573 match(RegVectMask);
5574 match(pRegGov);
5575 op_cost(0);
5576 format %{ %}
5577 interface(REG_INTER);
5578 %}
5579
5580 operand pRegGov()
5581 %{
5582 constraint(ALLOC_IN_RC(gov_pr));
5583 match(RegVectMask);
5584 op_cost(0);
5585 format %{ %}
5586 interface(REG_INTER);
5587 %}
5588
5589 // Flags register, used as output of signed compare instructions
5590
5591 // note that on AArch64 we also use this register as the output for
5592 // for floating point compare instructions (CmpF CmpD). this ensures
5593 // that ordered inequality tests use GT, GE, LT or LE none of which
5594 // pass through cases where the result is unordered i.e. one or both
8903 match(Set dst (CastVV dst));
8904
8905 size(0);
8906 format %{ "# castVV of $dst" %}
8907 ins_encode(/* empty encoding */);
8908 ins_cost(0);
8909 ins_pipe(pipe_class_empty);
8910 %}
8911
8912 instruct castVV(vReg dst)
8913 %{
8914 match(Set dst (CastVV dst));
8915
8916 size(0);
8917 format %{ "# castVV of $dst" %}
8918 ins_encode(/* empty encoding */);
8919 ins_cost(0);
8920 ins_pipe(pipe_class_empty);
8921 %}
8922
8923 instruct castVVMask(pRegGov dst)
8924 %{
8925 match(Set dst (CastVV dst));
8926
8927 size(0);
8928 format %{ "# castVV of $dst" %}
8929 ins_encode(/* empty encoding */);
8930 ins_cost(0);
8931 ins_pipe(pipe_class_empty);
8932 %}
8933
8934 // ============================================================================
8935 // Atomic operation instructions
8936 //
8937 // Intel and SPARC both implement Ideal Node LoadPLocked and
8938 // Store{PIL}Conditional instructions using a normal load for the
8939 // LoadPLocked and a CAS for the Store{PIL}Conditional.
8940 //
8941 // The ideal code appears only to use LoadPLocked/StorePLocked as a
8942 // pair to lock object allocations from Eden space when not using
8943 // TLABs.
8944 //
8945 // There does not appear to be a Load{IL}Locked Ideal Node and the
8946 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
8947 // and to use StoreIConditional only for 32-bit and StoreLConditional
8948 // only for 64-bit.
8949 //
8950 // We implement LoadPLocked and StorePLocked instructions using,
8951 // respectively the AArch64 hw load-exclusive and store-conditional
8952 // instructions. Whereas we must implement each of
8953 // Store{IL}Conditional using a CAS which employs a pair of
|