< prev index next >

src/hotspot/cpu/aarch64/aarch64.ad

Print this page
*** 2057,11 ***
    enum RC dst_hi_rc = rc_class(dst_hi);
    enum RC dst_lo_rc = rc_class(dst_lo);
  
    assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
  
!   if (src_hi != OptoReg::Bad) {
      assert((src_lo&1)==0 && src_lo+1==src_hi &&
             (dst_lo&1)==0 && dst_lo+1==dst_hi,
             "expected aligned-adjacent pairs");
    }
  
--- 2057,11 ---
    enum RC dst_hi_rc = rc_class(dst_hi);
    enum RC dst_lo_rc = rc_class(dst_lo);
  
    assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
  
!   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
      assert((src_lo&1)==0 && src_lo+1==src_hi &&
             (dst_lo&1)==0 && dst_lo+1==dst_hi,
             "expected aligned-adjacent pairs");
    }
  

*** 2072,11 ***
    bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
                (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
    int src_offset = ra_->reg2offset(src_lo);
    int dst_offset = ra_->reg2offset(dst_lo);
  
!   if (bottom_type()->isa_vect() != NULL) {
      uint ireg = ideal_reg();
      if (ireg == Op_VecA && cbuf) {
        C2_MacroAssembler _masm(cbuf);
        int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
        if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
--- 2072,11 ---
    bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
                (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
    int src_offset = ra_->reg2offset(src_lo);
    int dst_offset = ra_->reg2offset(dst_lo);
  
!   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
      uint ireg = ideal_reg();
      if (ireg == Op_VecA && cbuf) {
        C2_MacroAssembler _masm(cbuf);
        int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
        if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {

*** 2178,14 ***
        if (dst_lo_rc == rc_int) {  // stack --> gpr load
          __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
        } else if (dst_lo_rc == rc_float) { // stack --> fpr load
          __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
                     is64 ? __ D : __ S, src_offset);
        } else {                    // stack --> stack copy
          assert(dst_lo_rc == rc_stack, "spill to bad register class");
!         __ unspill(rscratch1, is64, src_offset);
!         __ spill(rscratch1, is64, dst_offset);
        }
        break;
      default:
        assert(false, "bad rc_class for spill");
        ShouldNotReachHere();
--- 2178,33 ---
        if (dst_lo_rc == rc_int) {  // stack --> gpr load
          __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
        } else if (dst_lo_rc == rc_float) { // stack --> fpr load
          __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
                     is64 ? __ D : __ S, src_offset);
+       } else if (dst_lo_rc == rc_predicate) {
+         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
+                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
        } else {                    // stack --> stack copy
          assert(dst_lo_rc == rc_stack, "spill to bad register class");
!         if (ideal_reg() == Op_RegVectMask) {
!           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
+                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
+         } else {
+           __ unspill(rscratch1, is64, src_offset);
+           __ spill(rscratch1, is64, dst_offset);
+         }
+       }
+       break;
+     case rc_predicate:
+       if (dst_lo_rc == rc_predicate) {
+         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
+       } else if (dst_lo_rc == rc_stack) {
+         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
+                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
+       } else {
+         assert(false, "bad src and dst rc_class combination.");
+         ShouldNotReachHere();
        }
        break;
      default:
        assert(false, "bad rc_class for spill");
        ShouldNotReachHere();

*** 2202,11 ***
      if (dst_lo_rc == rc_stack) {
        st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
      } else {
        st->print("%s", Matcher::regName[dst_lo]);
      }
!     if (bottom_type()->isa_vect() != NULL) {
        int vsize = 0;
        switch (ideal_reg()) {
        case Op_VecD:
          vsize = 64;
          break;
--- 2221,11 ---
      if (dst_lo_rc == rc_stack) {
        st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
      } else {
        st->print("%s", Matcher::regName[dst_lo]);
      }
!     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
        int vsize = 0;
        switch (ideal_reg()) {
        case Op_VecD:
          vsize = 64;
          break;

*** 2219,10 ***
--- 2238,14 ---
        default:
          assert(false, "bad register type for spill");
          ShouldNotReachHere();
        }
        st->print("\t# vector spill size = %d", vsize);
+     } else if (ideal_reg() == Op_RegVectMask) {
+       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
+       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
+       st->print("\t# predicate spill size = %d", vsize);
      } else {
        st->print("\t# spill size = %d", is64 ? 64 : 32);
      }
    }
  

*** 2378,10 ***
--- 2401,22 ---
      case Op_CacheWBPostSync:
        if (!VM_Version::supports_data_cache_line_flush()) {
          ret_value = false;
        }
        break;
+     case Op_LoadVectorMasked:
+     case Op_StoreVectorMasked:
+     case Op_LoadVectorGatherMasked:
+     case Op_StoreVectorScatterMasked:
+     case Op_MaskAll:
+     case Op_AndVMask:
+     case Op_OrVMask:
+     case Op_XorVMask:
+       if (UseSVE == 0) {
+         ret_value = false;
+       }
+       break;
    }
  
    return ret_value; // Per default match rules are supported.
  }
  

*** 2446,10 ***
--- 2481,19 ---
      }
    }
    return vector_size_supported(bt, vlen);
  }
  
+ const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
+   // Only SVE supports masked operations.
+   if (UseSVE == 0) {
+     return false;
+   }
+   return match_rule_supported(opcode) &&
+          masked_op_sve_supported(opcode, vlen, bt);
+ }
+ 
  const RegMask* Matcher::predicate_reg_mask(void) {
    return &_PR_REG_mask;
  }
  
  const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {

*** 2659,14 ***
    return true;
  }
  
  // Should the matcher clone input 'm' of node 'n'?
  bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
!   if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
!     mstack.push(m, Visit);           // m = ShiftCntV
      return true;
    }
    return false;
  }
  
  // Should the Matcher clone shifts on addressing modes, expecting them
  // to be subsumed into complex addressing expressions or compute them
--- 2703,18 ---
    return true;
  }
  
  // Should the matcher clone input 'm' of node 'n'?
  bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
!   // ShiftV src (ShiftCntV con)
!   // StoreVector (VectorStoreMask src)
+   if (is_vshift_con_pattern(n, m) ||
+       (UseSVE > 0 && m->Opcode() == Op_VectorStoreMask && n->Opcode() == Op_StoreVector)) {
+     mstack.push(m, Visit);
      return true;
    }
+ 
    return false;
  }
  
  // Should the Matcher clone shifts on addressing modes, expecting them
  // to be subsumed into complex addressing expressions or compute them

*** 5521,10 ***
--- 5569,11 ---
  
  operand pReg()
  %{
    constraint(ALLOC_IN_RC(pr_reg));
    match(RegVectMask);
+   match(pRegGov);
    op_cost(0);
    format %{ %}
    interface(REG_INTER);
  %}
  

*** 8869,10 ***
--- 8918,21 ---
    ins_encode(/* empty encoding */);
    ins_cost(0);
    ins_pipe(pipe_class_empty);
  %}
  
+ instruct castVVMask(pRegGov dst)
+ %{
+   match(Set dst (CastVV dst));
+ 
+   size(0);
+   format %{ "# castVV of $dst" %}
+   ins_encode(/* empty encoding */);
+   ins_cost(0);
+   ins_pipe(pipe_class_empty);
+ %}
+ 
  // ============================================================================
  // Atomic operation instructions
  //
  // Intel and SPARC both implement Ideal Node LoadPLocked and
  // Store{PIL}Conditional instructions using a normal load for the
< prev index next >