src/hotspot/share/opto/matcher.cpp

*** 432,10 ***
--- 432,28 ---
    rms[TypeFunc::ReturnAdr] = ret_adr;
    rms[TypeFunc::FramePtr ] = fp;
    return rms;
  }
  
+ const int Matcher::scalable_predicate_reg_slots() {
+   assert(Matcher::has_predicated_vectors() && Matcher::supports_scalable_vector(),
+         "scalable predicate vector should be supported");
+   int vector_reg_bit_size = Matcher::scalable_vector_reg_size(T_BYTE) << LogBitsPerByte;
+   // We assume each predicate register is one-eighth of the size of
+   // scalable vector register, one mask bit per vector byte.
+   int predicate_reg_bit_size = vector_reg_bit_size >> 3;
+   // Compute number of slots which is required when scalable predicate
+   // register is spilled. E.g. if scalable vector register is 640 bits,
+   // predicate register is 80 bits, which is 2.5 * slots.
+   // We will round up the slot number to power of 2, which is required
+   // by find_first_set().
+   int slots = predicate_reg_bit_size & (BitsPerInt - 1)
+               ? (predicate_reg_bit_size >> LogBitsPerInt) + 1
+               : predicate_reg_bit_size >> LogBitsPerInt;
+   return round_up_power_of_2(slots);
+ }
+ 
  #define NOF_STACK_MASKS (3*13)
  
  // Create the initial stack mask used by values spilling to the stack.
  // Disallow any debug info in outgoing argument areas by setting the
  // initial mask accordingly.

*** 540,10 ***
--- 558,12 ---
     idealreg2spillmask[Op_RegD]->OR(aligned_stack_mask);
  
    if (Matcher::has_predicated_vectors()) {
      *idealreg2spillmask[Op_RegVectMask] = *idealreg2regmask[Op_RegVectMask];
       idealreg2spillmask[Op_RegVectMask]->OR(aligned_stack_mask);
+   } else {
+     *idealreg2spillmask[Op_RegVectMask] = RegMask::Empty;
    }
  
    if (Matcher::vector_size_supported(T_BYTE,4)) {
      *idealreg2spillmask[Op_VecS] = *idealreg2regmask[Op_VecS];
       idealreg2spillmask[Op_VecS]->OR(C->FIRST_STACK_mask());

*** 612,10 ***
--- 632,23 ---
    }
  
    if (Matcher::supports_scalable_vector()) {
      int k = 1;
      OptoReg::Name in = OptoReg::add(_in_arg_limit, -1);
+     // Exclude last input arg stack slots to avoid spilling vector register there,
+     // otherwise RegVectMask spills could stomp over stack slots in caller frame.
+     for (; (in >= init_in) && (k < scalable_predicate_reg_slots()); k++) {
+       scalable_stack_mask.Remove(in);
+       in = OptoReg::add(in, -1);
+     }
+ 
+     // For RegVectMask
+     scalable_stack_mask.clear_to_sets(scalable_predicate_reg_slots());
+     assert(scalable_stack_mask.is_AllStack(), "should be infinite stack");
+     *idealreg2spillmask[Op_RegVectMask] = *idealreg2regmask[Op_RegVectMask];
+     idealreg2spillmask[Op_RegVectMask]->OR(scalable_stack_mask);
+ 
      // Exclude last input arg stack slots to avoid spilling vector register there,
      // otherwise vector spills could stomp over stack slots in caller frame.
      for (; (in >= init_in) && (k < scalable_vector_reg_size(T_FLOAT)); k++) {
        scalable_stack_mask.Remove(in);
        in = OptoReg::add(in, -1);

*** 2226,10 ***
--- 2259,11 ---
      case Op_FmaVD:
      case Op_FmaVF:
      case Op_MacroLogicV:
      case Op_LoadVectorMasked:
      case Op_VectorCmpMasked:
+     case Op_VectorLoadMask:
        set_shared(n); // Force result into register (it will be anyways)
        break;
      case Op_ConP: {  // Convert pointers above the centerline to NUL
        TypeNode *tn = n->as_Type(); // Constants derive from type nodes
        const TypePtr* tp = tn->type()->is_ptr();

*** 2271,10 ***
--- 2305,25 ---
    } // end_switch
    return false;
  }
  
  void Matcher::find_shared_post_visit(Node* n, uint opcode) {
+   if (n->is_predicated_vector()) {
+     // Restructure into binary trees for Matching.
+     if (n->req() == 4) {
+       n->set_req(1, new BinaryNode(n->in(1), n->in(2)));
+       n->set_req(2, n->in(3));
+       n->del_req(3);
+     } else if (n->req() == 5) {
+       n->set_req(1, new BinaryNode(n->in(1), n->in(2)));
+       n->set_req(2, new BinaryNode(n->in(3), n->in(4)));
+       n->del_req(4);
+       n->del_req(3);
+     }
+     return;
+   }
+ 
    switch(opcode) {       // Handle some opcodes special
      case Op_StorePConditional:
      case Op_StoreIConditional:
      case Op_StoreLConditional:
      case Op_CompareAndExchangeB:

*** 2410,16 ***
--- 2459,26 ---
        n->set_req(1, pair);
        n->set_req(2, n->in(3));
        n->del_req(3);
        break;
      }
+     case Op_LoadVectorGatherMasked:
      case Op_StoreVectorScatter: {
        Node* pair = new BinaryNode(n->in(MemNode::ValueIn), n->in(MemNode::ValueIn+1));
        n->set_req(MemNode::ValueIn, pair);
        n->del_req(MemNode::ValueIn+1);
        break;
      }
+     case Op_StoreVectorScatterMasked: {
+       Node* pair = new BinaryNode(n->in(MemNode::ValueIn+1), n->in(MemNode::ValueIn+2));
+       n->set_req(MemNode::ValueIn+1, pair);
+       n->del_req(MemNode::ValueIn+2);
+       pair = new BinaryNode(n->in(MemNode::ValueIn), n->in(MemNode::ValueIn+1));
+       n->set_req(MemNode::ValueIn, pair);
+       n->del_req(MemNode::ValueIn+1);
+       break;
+     }
      case Op_VectorMaskCmp: {
        n->set_req(1, new BinaryNode(n->in(1), n->in(2)));
        n->set_req(2, n->in(3));
        n->del_req(3);
        break;