< prev index next > src/hotspot/cpu/aarch64/aarch64.ad
Print this page
enum RC dst_hi_rc = rc_class(dst_hi);
enum RC dst_lo_rc = rc_class(dst_lo);
assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
! if (src_hi != OptoReg::Bad) {
assert((src_lo&1)==0 && src_lo+1==src_hi &&
(dst_lo&1)==0 && dst_lo+1==dst_hi,
"expected aligned-adjacent pairs");
}
enum RC dst_hi_rc = rc_class(dst_hi);
enum RC dst_lo_rc = rc_class(dst_lo);
assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
! if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
assert((src_lo&1)==0 && src_lo+1==src_hi &&
(dst_lo&1)==0 && dst_lo+1==dst_hi,
"expected aligned-adjacent pairs");
}
bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
(dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
int src_offset = ra_->reg2offset(src_lo);
int dst_offset = ra_->reg2offset(dst_lo);
! if (bottom_type()->isa_vect() != NULL) {
uint ireg = ideal_reg();
if (ireg == Op_VecA && cbuf) {
C2_MacroAssembler _masm(cbuf);
int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
(dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
int src_offset = ra_->reg2offset(src_lo);
int dst_offset = ra_->reg2offset(dst_lo);
! if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
uint ireg = ideal_reg();
if (ireg == Op_VecA && cbuf) {
C2_MacroAssembler _masm(cbuf);
int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
if (dst_lo_rc == rc_int) { // stack --> gpr load
__ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
} else if (dst_lo_rc == rc_float) { // stack --> fpr load
__ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
is64 ? __ D : __ S, src_offset);
} else { // stack --> stack copy
assert(dst_lo_rc == rc_stack, "spill to bad register class");
! __ unspill(rscratch1, is64, src_offset);
! __ spill(rscratch1, is64, dst_offset);
}
break;
default:
assert(false, "bad rc_class for spill");
ShouldNotReachHere();
if (dst_lo_rc == rc_int) { // stack --> gpr load
__ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
} else if (dst_lo_rc == rc_float) { // stack --> fpr load
__ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
is64 ? __ D : __ S, src_offset);
+ } else if (dst_lo_rc == rc_predicate) {
+ __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
+ Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
} else { // stack --> stack copy
assert(dst_lo_rc == rc_stack, "spill to bad register class");
! if (ideal_reg() == Op_RegVectMask) {
! __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
+ Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
+ } else {
+ __ unspill(rscratch1, is64, src_offset);
+ __ spill(rscratch1, is64, dst_offset);
+ }
+ }
+ break;
+ case rc_predicate:
+ if (dst_lo_rc == rc_predicate) {
+ __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
+ } else if (dst_lo_rc == rc_stack) {
+ __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
+ Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
+ } else {
+ assert(false, "bad src and dst rc_class combination.");
+ ShouldNotReachHere();
}
break;
default:
assert(false, "bad rc_class for spill");
ShouldNotReachHere();
if (dst_lo_rc == rc_stack) {
st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
} else {
st->print("%s", Matcher::regName[dst_lo]);
}
! if (bottom_type()->isa_vect() != NULL) {
int vsize = 0;
switch (ideal_reg()) {
case Op_VecD:
vsize = 64;
break;
if (dst_lo_rc == rc_stack) {
st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
} else {
st->print("%s", Matcher::regName[dst_lo]);
}
! if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
int vsize = 0;
switch (ideal_reg()) {
case Op_VecD:
vsize = 64;
break;
default:
assert(false, "bad register type for spill");
ShouldNotReachHere();
}
st->print("\t# vector spill size = %d", vsize);
+ } else if (ideal_reg() == Op_RegVectMask) {
+ assert(Matcher::supports_scalable_vector(), "bad register type for spill");
+ int vsize = Matcher::scalable_predicate_reg_slots() * 32;
+ st->print("\t# predicate spill size = %d", vsize);
} else {
st->print("\t# spill size = %d", is64 ? 64 : 32);
}
}
case Op_CacheWBPostSync:
if (!VM_Version::supports_data_cache_line_flush()) {
ret_value = false;
}
break;
+ case Op_LoadVectorMasked:
+ case Op_StoreVectorMasked:
+ case Op_LoadVectorGatherMasked:
+ case Op_StoreVectorScatterMasked:
+ case Op_MaskAll:
+ case Op_AndVMask:
+ case Op_OrVMask:
+ case Op_XorVMask:
+ if (UseSVE == 0) {
+ ret_value = false;
+ }
+ break;
}
return ret_value; // Per default match rules are supported.
}
}
}
return vector_size_supported(bt, vlen);
}
+ const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
+ // Only SVE supports masked operations.
+ if (UseSVE == 0) {
+ return false;
+ }
+ return match_rule_supported(opcode) &&
+ masked_op_sve_supported(opcode, vlen, bt);
+ }
+
const RegMask* Matcher::predicate_reg_mask(void) {
return &_PR_REG_mask;
}
const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
return true;
}
// Should the matcher clone input 'm' of node 'n'?
bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
! if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
! mstack.push(m, Visit); // m = ShiftCntV
return true;
}
return false;
}
// Should the Matcher clone shifts on addressing modes, expecting them
// to be subsumed into complex addressing expressions or compute them
return true;
}
// Should the matcher clone input 'm' of node 'n'?
bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
! // ShiftV src (ShiftCntV con)
! // StoreVector (VectorStoreMask src)
+ if (is_vshift_con_pattern(n, m) ||
+ (UseSVE > 0 && m->Opcode() == Op_VectorStoreMask && n->Opcode() == Op_StoreVector)) {
+ mstack.push(m, Visit);
return true;
}
+
return false;
}
// Should the Matcher clone shifts on addressing modes, expecting them
// to be subsumed into complex addressing expressions or compute them
operand pReg()
%{
constraint(ALLOC_IN_RC(pr_reg));
match(RegVectMask);
+ match(pRegGov);
op_cost(0);
format %{ %}
interface(REG_INTER);
%}
ins_encode(/* empty encoding */);
ins_cost(0);
ins_pipe(pipe_class_empty);
%}
+ instruct castVVMask(pRegGov dst)
+ %{
+ match(Set dst (CastVV dst));
+
+ size(0);
+ format %{ "# castVV of $dst" %}
+ ins_encode(/* empty encoding */);
+ ins_cost(0);
+ ins_pipe(pipe_class_empty);
+ %}
+
// ============================================================================
// Atomic operation instructions
//
// Intel and SPARC both implement Ideal Node LoadPLocked and
// Store{PIL}Conditional instructions using a normal load for the
< prev index next >